[ci] Add missing components to CI

.gitignore

@@ -11,3 +11,5 @@ polkadot_argument_parsing
 .env
 **/._*
 
+cargo_home
+cargo_target_dir

.gitlab-ci.yml

@@ -1,32 +1,9 @@
-# .gitlab-ci.yml
+# polkadot-sdk | CI definitions (via GitLab CI)
 #
-# substrate
+# FYI: Pipelines can be triggered manually through the web UI (if you have enough permissions)
 #
-# pipelines can be triggered manually in the web
-#
-# Currently the file is divided into subfiles. Each stage has a different file which
-# can be found here: gitlab/pipeline/<stage_name>.yml
-#
-# Instead of YAML anchors "extends" is used.
-# Useful links:
-#    https://docs.gitlab.com/ee/ci/yaml/index.html#extends
-#    https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags
-#
-# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a
-# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/
-#
-# my-example-job:
-#   stage:                           test # One of the stages listed below this job (required)
-#   image:                           paritytech/tools:latest # Any docker image (required)
-#   allow_failure:                   true # Allow the pipeline to continue if this job fails (default: false)
-#   needs:
-#     - job:                         test-linux # Any jobs that are required to run before this job (optional)
-#   variables:
-#     MY_ENVIRONMENT_VARIABLE:       "some useful value" # Environment variables passed to the job (optional)
-#   script:
-#     - echo "List of shell commands to run in your job"
-#     - echo "You can also just specify a script here, like so:"
-#     - ./gitlab/my_amazing_script.sh
+# Currently, entire CI instructions are split into different subfiles. Each CI stage has a corresponding
+# file which can be found here: .gitlab/pipeline/<stage_name>.yml
 
 stages:
   - check
@@ -100,7 +77,7 @@ default:
 .kubernetes-env:
   image: "${CI_IMAGE}"
   before_script:
-    # - !reference [.job-switcher, before_script]
+    - !reference [.job-switcher, before_script]
     - !reference [.prepare-env, before_script]
   tags:
     - kubernetes-parity-build
@@ -112,6 +89,8 @@ default:
     - rustup +nightly show
     - cargo +nightly --version
 
+# collecting vars for pipeline stopper
+# they will be used if the job fails
 .pipeline-stopper-vars:
   script:
     - !reference [.job-switcher, before_script]
@@ -133,23 +112,21 @@ default:
     - !reference [.rust-info-script, script]
     - !reference [.rusty-cachier, before_script]
     - !reference [.pipeline-stopper-vars, script]
-  after_script:
-    - !reference [.rusty-cachier, after_script]
   tags:
     - linux-docker-vm-c2
 
 # rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier
-# Description of the commands is available here - https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
+# rusty-cachier's commands are described here: https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
 .rusty-cachier:
   before_script:
-    # - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash
-    # - rusty-cachier environment check --gracefully
-    # - $(rusty-cachier environment inject)
-    # - rusty-cachier project mtime
-    - echo tbd
-  after_script:
-    - echo tbd
-    # - env RUSTY_CACHIER_SUPRESS_OUTPUT=true rusty-cachier snapshot destroy
+    - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash
+    - mkdir -p cargo_home cargo_target_dir
+    - export CARGO_HOME=$CI_PROJECT_DIR/cargo_home
+    - export CARGO_TARGET_DIR=$CI_PROJECT_DIR/cargo_target_dir
+    - find . \( -path ./cargo_target_dir -o -path ./cargo_home \) -prune -o -type f -exec touch -t 202005260100 {} +
+    - git restore-mtime
+    - rusty-cachier --version
+    - rusty-cachier project touch-changed
 
 .common-refs:
   rules:
@@ -233,6 +210,126 @@ include:
   # zombienet jobs
   - .gitlab/pipeline/zombienet.yml
   # # timestamp handler
-  # - project: parity/infrastructure/ci_cd/shared
-  #   ref: v0.2
-  #   file: /common/timestamp.yml
+  - project: parity/infrastructure/ci_cd/shared
+    ref: v0.2
+    file: /common/timestamp.yml
+
+# This job cancels the whole pipeline if any of provided jobs fail.
+# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
+# to fail the pipeline as soon as possible to shorten the feedback loop.
+.cancel-pipeline-template:
+  stage: .post
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
+      when: on_failure
+  variables:
+    PROJECT_ID: "${CI_PROJECT_ID}"
+    PROJECT_NAME: "${CI_PROJECT_NAME}"
+    PIPELINE_ID: "${CI_PIPELINE_ID}"
+    FAILED_JOB_URL: "${FAILED_JOB_URL}"
+    FAILED_JOB_NAME: "${FAILED_JOB_NAME}"
+    PR_NUM: "${PR_NUM}"
+  trigger:
+    project: "parity/infrastructure/ci_cd/pipeline-stopper"
+    branch: "as-improve"
+
+remove-cancel-pipeline-message:
+  stage: .post
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
+  variables:
+    PROJECT_ID: "${CI_PROJECT_ID}"
+    PROJECT_NAME: "${CI_PROJECT_NAME}"
+    PIPELINE_ID: "${CI_PIPELINE_ID}"
+    FAILED_JOB_URL: "https://gitlab.com"
+    FAILED_JOB_NAME: "nope"
+    PR_NUM: "${CI_COMMIT_REF_NAME}"
+  trigger:
+    project: "parity/infrastructure/ci_cd/pipeline-stopper"
+
+# need to copy jobs this way because otherwise gitlab will wait
+# for all 3 jobs to finish instead of cancelling if one fails
+cancel-pipeline-test-linux-stable1:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "test-linux-stable 1/3"
+
+cancel-pipeline-test-linux-stable2:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "test-linux-stable 2/3"
+
+cancel-pipeline-test-linux-stable3:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "test-linux-stable 3/3"
+
+cancel-pipeline-test-linux-stable-additional-tests:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "test-linux-stable-additional-tests"
+
+cancel-pipeline-test-linux-stable-slow:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "test-linux-stable-slow"
+
+cancel-pipeline-cargo-check-benches1:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-benches 1/2"
+
+cancel-pipeline-cargo-check-benches2:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-benches 2/2"
+
+cancel-pipeline-test-linux-stable-int:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: test-linux-stable-int
+
+cancel-pipeline-cargo-check-each-crate-1:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 1/6"
+
+cancel-pipeline-cargo-check-each-crate-2:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 2/6"
+
+cancel-pipeline-cargo-check-each-crate-3:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 3/6"
+
+cancel-pipeline-cargo-check-each-crate-4:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 4/6"
+
+cancel-pipeline-cargo-check-each-crate-5:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 5/6"
+
+cancel-pipeline-cargo-check-each-crate-6:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 6/6"
+
+cancel-pipeline-cargo-check-each-crate-macos:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: cargo-check-each-crate-macos
+
+cancel-pipeline-check-tracing:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: check-tracing
+
+cancel-pipeline-cargo-clippy:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: cargo-clippy

.gitlab/pipeline/check.yml

@@ -43,7 +43,6 @@ cargo-deny-licenses:
   script:
     - $CARGO_DENY_CMD --hide-inclusion-graph
   after_script:
-    # - !reference [.rusty-cachier, after_script]
     - echo "___The complete log is in the artifacts___"
     - $CARGO_DENY_CMD 2> deny.log
     - if [ $CI_JOB_STATUS != 'success' ]; then