Merge branch 'master' into 3x8_update_nightly_fix_fmt

* master: (71 commits)
  Remove light client companion (#4191)
  Enable full use of pallet-bags-list in westend and kusama runtimes (#4195)
  collator-protocol: do not connect to the next group (#4261)
  Per subsystem CPU usage tracking (#4239)
  CI: chore (#3957)
  Companion – Update jsonrpsee to 0.4.1 (#4256)
  Add more XCM tracing (#4211)
  Update dependencies for latest substrate master (#4258)
  Bump mick-jaeger from 0.1.4 to 0.1.6 (#4249)
  Bump dlmalloc from 0.2.2 to 0.2.3 (#4250)
  Bump libc from 0.2.106 to 0.2.107 (#4235)
  Bump paste from 1.0.5 to 1.0.6 (#4244)
  Bump serde_json from 1.0.68 to 1.0.69 (#4236)
  Update `wasmtime` and related dependencies (companion for Substrate#10149) (#4210)
  update cargo lock to unbreak dep of a dep (#4245)
  Increase maximum chunk size to adjust for small networks. (#4220)
  availability recovery type name clarifications (#4203)
  Update `bridge/` codeowners (#4222)
  fix(staking miner): use `StorageKey` in getStorage (#4231)
  Change path for the tests to master (#4223)
  ...

.github/CODEOWNERS

@@ -1 +1,2 @@
-bridges/ @tomusdrw @svyatonik @hcastano
+bridges/ @tomusdrw @svyatonik @adoerr @acatangiu @antonio-dropulic
+

.github/workflows/check-labels.yml

@@ -8,7 +8,8 @@ jobs:
   check-labels:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout sources
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.ref }}

.github/workflows/extrinsic-ordering-check-from-bin.yml

@@ -27,7 +27,8 @@ jobs:
       REF_URL: ${{github.event.inputs.reference_url}}
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout sources
+        uses: actions/checkout@v2
 
       - name: Fetch binary
         run: |

.github/workflows/publish-docker-release.yml

@@ -9,7 +9,8 @@ jobs:
   main:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout sources
+        uses: actions/checkout@v2
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Cache Docker layers

.github/workflows/publish-draft-release.yml

@@ -26,7 +26,8 @@ jobs:
       matrix:
         runtime: ["polkadot", "kusama", "westend"]
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout sources
+        uses: actions/checkout@v2
       - name: Cache target dir
         uses: actions/cache@v2
         with:
@@ -64,7 +65,8 @@ jobs:
       release_url: ${{ steps.create-release.outputs.html_url }}
       asset_upload_url: ${{ steps.create-release.outputs.upload_url }}
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout sources
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
           path: polkadot
@@ -100,8 +102,10 @@ jobs:
       matrix:
         runtime: ["polkadot", "kusama", "westend"]
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
+      - name: Checkout sources
+        uses: actions/checkout@v2
+      - name: Download artifacts
+        uses: actions/download-artifact@v2
       - name: Set up Ruby 2.7
         uses: actions/setup-ruby@v1
         with:
@@ -128,7 +132,7 @@ jobs:
     needs: publish-draft-release
     steps:
       - name: Internal polkadot channel
-        uses: s3krit/matrix-message-action@v0.0.2
+        uses: s3krit/matrix-message-action@v0.0.3
         with:
           room_id: ${{ secrets.INTERNAL_POLKADOT_MATRIX_ROOM_ID }}
           access_token: ${{ secrets.MATRIX_ACCESS_TOKEN }}

.github/workflows/release-bot.yml

@@ -15,7 +15,8 @@ jobs:
         - '!NTogofoetwjbTwOoPi:matrix.parity.io' # Internal release-notes channel
     runs-on: ubuntu-latest
     steps:
-      - uses: s3krit/matrix-message-action@v0.0.3
+      - name: Send Matrix message
+        uses: s3krit/matrix-message-action@v0.0.3
         with:
           room_id: ${{ matrix.channel }} # heh
           access_token: ${{ secrets.MATRIX_ACCESS_TOKEN }}

.github/workflows/release-candidate.yml

@@ -7,7 +7,8 @@ jobs:
   tag_rc:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout sources
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
       - id: compute_tag
@@ -48,7 +49,8 @@ jobs:
           VERSION: ${{ steps.compute_tag.outputs.version }}
         with:
           filename: .github/ISSUE_TEMPLATE/release.md
-      - uses: s3krit/matrix-message-action@v0.0.2
+      - name: Send Matrix message
+        uses: s3krit/matrix-message-action@v0.0.3
         if: steps.create-issue.outputs.url != ''
         with:
           room_id: ${{ secrets.INTERNAL_POLKADOT_MATRIX_ROOM_ID }}

.gitlab-ci.yml

@@ -6,7 +6,7 @@
 # setting DEPLOY_TAG will only deploy the tagged image
 #
 # please do not add new jobs without "rules:" and "*-env". There are &rules-test for everything,
-# &rules-pr-only preset. And "kubernetes-env" with "docker-env" to set a runner
+# &rules-test-and-rococo preset. And "kubernetes-env" with "docker-env" to set a runner
 # which executes the job.
 
 stages:
@@ -15,8 +15,6 @@ stages:
   - publish
   - deploy
 
-image:                             paritytech/ci-linux:staging 
-
 workflow:
   rules:
     - if: $CI_COMMIT_TAG
@@ -26,11 +24,13 @@ variables:
   GIT_STRATEGY:                    fetch
   GIT_DEPTH:                       100
   CI_SERVER_NAME:                  "GitLab CI"
+  CI_IMAGE:                        "paritytech/ci-linux:staging"
   DOCKER_OS:                       "debian:stretch"
   ARCH:                            "x86_64"
   VAULT_SERVER_URL:                "https://vault.parity-mgmt-vault.parity.io"
   VAULT_AUTH_PATH:                 "gitlab-parity-io-jwt"
   VAULT_AUTH_ROLE:                 "cicd_gitlab_parity_${CI_PROJECT_NAME}"
+  SIMNET_IMAGE:                    "europe-west3-docker.pkg.dev/parity-simnet/simnet-images/simnet:v14"
   PIPELINE_SCRIPTS_TAG:            "v0.1"
 
 default:
@@ -56,6 +56,7 @@ default:
     - kubernetes-parity-build
 
 .docker-env:                       &docker-env
+  image:                           "${CI_IMAGE}"
   retry:
     max: 2
     when:
@@ -77,16 +78,16 @@ default:
   rules:
     - if: $CI_COMMIT_REF_NAME == "rococo-v1"
       when: never
-    - when: always
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
 
-.pr-only:                          &rules-pr-only
-  # these jobs run only on PRs
+.rules-test-and-rococo:            &rules-test-and-rococo
   rules:
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
-      when: never
     - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: never
+    - if: $CI_COMMIT_REF_NAME == "master"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
 
 #### Vault secrets
 .vault-secrets:                    &vault-secrets
@@ -143,7 +144,7 @@ check-runtime:
   stage:                           test
   image:                           paritytech/tools:latest
   <<:                              *kubernetes-env
-  <<:                              *rules-pr-only
+  <<:                              *rules-test
   <<:                              *vault-secrets
   variables:
     GITLAB_API:                    "https://gitlab.parity.io/api/v4"
@@ -152,19 +153,10 @@ check-runtime:
     - ./scripts/gitlab/check_runtime.sh
   allow_failure:                   true
 
-check-line-width:
-  stage:                           test
-  image:                           paritytech/tools:latest
-  <<:                              *kubernetes-env
-  <<:                              *rules-pr-only
-  script:
-    - ./scripts/gitlab/check_line_width.sh
-  allow_failure:                   true
-
 cargo-fmt:
   stage:                           test
-  <<:                              *kubernetes-env
-  <<:                              *rules-pr-only
+  <<:                              *docker-env
+  <<:                              *rules-test
   script:
     - cargo +nightly fmt --all -- --check
   allow_failure:                   true
@@ -198,7 +190,7 @@ test-build-linux-stable:
   script:
     - ./scripts/gitlab/test_linux_stable.sh
     # we're using the bin built here, instead of having a parallel `build-linux-release`
-    - time cargo build --release --verbose --bin polkadot
+    - time cargo build --release --verbose --bin polkadot --features "disputes"
     - sccache -s
     # pack artifacts
     - mkdir -p ./artifacts
@@ -238,11 +230,12 @@ check-no-default-features:
 spellcheck:
   stage:                           test
   <<:                              *docker-env
-  <<:                              *rules-pr-only
+  <<:                              *rules-test
   script:
     - cargo spellcheck --version
     # compare with the commit parent to the PR, given it's from a default branch
     - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH}
+    - echo "___Spellcheck is going to check your diff___"
     - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
     - time cargo spellcheck check -vvv --cfg=scripts/gitlab/spellcheck.toml --checkers hunspell --code 1
         $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
@@ -253,11 +246,7 @@ build-adder-collator:
   <<:                              *collect-artifacts
   <<:                              *docker-env
   <<:                              *compiler-info
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
+  <<:                              *rules-test-and-rococo
   script:
     - time cargo build --release --verbose -p test-parachain-adder-collator
     - sccache -s
@@ -286,26 +275,26 @@ build-adder-collator:
         --polkadot
         "$DEPENDENT_REPO"
         "$GITHUB_PR_TOKEN"
-    - cd "$DEPENDENT_REPO" && git rev-parse --abbrev-ref HEAD
 
 check-dependent-cumulus:
   <<: *check-dependent-project
   variables:
     DEPENDENT_REPO: cumulus
 
 check-transaction-versions:
-  # image must be ubuntu:20.04 based to match the linkers, this image has npm installed
-  image:                           paritytech/contracts-ci-linux:production
   stage:                           build
   <<:                              *rules-test
   <<:                              *docker-env
   <<:                              *vault-secrets
+  image:                           node:15
   needs:
     - job:                         test-build-linux-stable
       artifacts:                   true
   before_script:
     - apt-get -y update; apt-get -y install jq lsof
     - npm install --ignore-scripts -g @polkadot/metadata-cmp
+    # Set git config
+    - git config remote.origin.url "https://github.com/paritytech/polkadot.git"
     - git fetch origin release
   script:
     - scripts/gitlab/check_extrinsics_ordering.sh
@@ -334,7 +323,7 @@ build-rustdoc:
     - ./crate-docs/
   script:
     # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
-    - time cargo doc --no-deps --workspace --verbose
+    - time cargo doc --workspace --verbose
     - rm -f ./target/doc/.lock
     - mv ./target/doc ./crate-docs
     # FIXME: remove me after CI image gets nonroot
@@ -416,16 +405,12 @@ publish-adder-collator-image:
   # service image for Simnet
   stage:                           build
   <<:                              *build-push-image
+  <<:                              *rules-test-and-rococo
   variables:
     <<:                            *image-variables
     # scripts/dockerfiles/collator_injected.Dockerfile
     DOCKERFILE:                    dockerfiles/collator_injected.Dockerfile
     IMAGE_NAME:                    docker.io/paritypr/colander
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
   needs:
     - job:                         build-adder-collator
       artifacts:                   true
@@ -439,22 +424,20 @@ publish-adder-collator-image:
       # this artifact is used in simnet-tests job
       dotenv: ./artifacts/collator.env
 
-.update_weights:                   &update-weights
+update_polkadot_weights:           &update-weights
   stage:                           build
   when:                            manual
-  tags:
-    - weights
   variables:
     RUNTIME:                       polkadot
   artifacts:
     paths:
       - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
-  script: |
-    ./scripts/run_benches_for_runtime.sh $RUNTIME
-    git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
-
-update_polkadot_weights:
-  <<:                              *update-weights
+  script:
+    - ./scripts/run_benches_for_runtime.sh $RUNTIME
+    - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
+  # uses the "shell" executors
+  tags:
+    - weights
 
 update_kusama_weights:
   <<:                              *update-weights
@@ -547,6 +530,7 @@ publish-rustdoc:
     - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" ||
         echo "___Nothing to commit___"
     - git push origin gh-pages --force
+    - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___"
   after_script:
     - rm -rf .git/ ./*
 
@@ -555,7 +539,6 @@ publish-rustdoc:
 deploy-polkasync-kusama:
   stage:                           deploy
   rules:
-    # former .rules-build
     - if: $CI_COMMIT_REF_NAME == "rococo-v1"
       when: never
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -568,13 +551,11 @@ deploy-polkasync-kusama:
 
 simnet-tests:
   stage:                           deploy
-  image:                           "docker.io/paritytech/simnet:${SIMNET_REF}"
+  image:                           "${SIMNET_IMAGE}"
   <<:                              *kubernetes-env
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
+  <<:                              *rules-test-and-rococo
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/polkadot/tree/master/simnet_tests"
   needs:
     - job:                         publish-polkadot-image
     - job:                         publish-adder-collator-image
@@ -586,17 +567,19 @@ simnet-tests:
 
   # SIMNET_REF is a gitlab variable
   before_script:
-    - echo "Simnet Tests Config
-        ${SIMNET_IMAGE_NAME} ${SIMNET_REF}
-        ${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}
-        ${COLLATOR_IMAGE_NAME} ${COLLATOR_IMAGE_TAG}"
+    - echo "Simnet Tests Config"
+    - echo "${SIMNET_IMAGE}"
+    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
+    - echo "${COLLATOR_IMAGE_NAME} ${COLLATOR_IMAGE_TAG}"
+    - echo "${GH_DIR}"
   script:
-    - /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh
-        --github-remote-dir="https://github.com/paritytech/polkadot/tree/master/simnet_tests"
-        --config="simnet_tests/configs/simple_rococo_testnet.toml"
-        --image="${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}"
-        --image-2="${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}"
-        --image-simnet="docker.io/paritytech/simnet:${SIMNET_REF}"
+    - /home/nonroot/simnet/scripts/run-test-environment-manager-v2.sh
+        --github-remote-dir="${GH_DIR}"
+        --tag=smoketest
+        --image="PARACHAINSIMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}"
+        --image="SYNTHIMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}"
+        --image="COLIMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}"
+        --image="SCRIPTSIMAGE=${SIMNET_IMAGE}"
   allow_failure:                   true
   retry: 2
   tags: