Merge branch 'main' into xdm_mmr

.github/workflows/domain-genesis-storage-snapshot-build.yml

@@ -0,0 +1,49 @@
+# This action enabling building domain genesis storage used in registering new domain runtime, can be triggered manually or by release creation.
+#
+# Domain genesis storages are built for releases and for manually triggered runs, uploaded to artifacts and assets.
+name: Domain genesis storage snapshot build
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - 'domain-genesis-storage-snapshot-*'
+      - 'domain-genesis-storage-gemini-*'
+
+jobs:
+  domain-genesis-storage:
+    runs-on: ${{ fromJson(github.repository_owner == 'subspace' && '["self-hosted", "ubuntu-20.04-x86-64"]' || 'ubuntu-22.04') }}
+    permissions:
+      contents: write
+      packages: write
+
+    steps:
+      - name: Build node image
+        id: build
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 # @v3.2.0
+        with:
+          file: Dockerfile-node
+          push: false
+
+      - name: Generate testnet domain genesis storages
+        run: |
+          docker run --rm -u root ${{ steps.build.outputs.digest }} domain build-genesis-storage --chain gemini-3h > domain-genesis-storage-gemini-3h
+          docker run --rm -u root ${{ steps.build.outputs.digest }} domain build-genesis-storage --chain devnet > domain-genesis-storage-devnet
+
+      - name: Upload domain genesis storages to artifacts
+        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # @v3.1.1
+        with:
+          name: domain-genesis-storage
+          path: |
+            domain-genesis-storage-gemini-3h
+            domain-genesis-storage-devnet
+          if-no-files-found: error
+
+      - name: Upload domain genesis storages to assets
+        uses: alexellis/upload-assets@259de5111cb56966d046ced998941e93f91d2c93 # @0.4.0
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          asset_paths: '["domain-genesis-storage-gemini-3h", "domain-genesis-storage-devnet"]'
+        # Only run for releases
+        if: github.event_name == 'push' && github.ref_type == 'tag'

.github/workflows/snapshot-build.yml

@@ -86,7 +86,7 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           event-type: trivy-scan-dispatch
-          client-payload: '{"sha": "${{ github.sha }}"}'
+          client-payload: '{"image": "ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:sha-${{ github.sha }}"}'
 
   executables:
     strategy:

.github/workflows/trivy-security-scan.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       - name: Check Docker image availability with retry
         run: |
-          image="ghcr.io/${{ github.repository_owner }}/${{ github.event.client_payload.image }}:${{ github.event.client_payload.sha }}"
+          image="${{ github.event.client_payload.image }}"
           timeout=900 # Timeout in seconds (15 minutes)
           interval=300 # Interval between retries in seconds (5 minutes)
           retry_limit=5 # Number of retries
@@ -43,12 +43,6 @@ jobs:
     needs: wait-for-image
     if: needs.wait-for-image.outputs.image-available == 'true'
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        image:
-          - farmer
-          - node
-          - bootstrap-node
 
     steps:
       - name: Checkout code
@@ -57,7 +51,7 @@ jobs:
       - name: Run Trivy vulnerability scanner on image
         uses: aquasecurity/trivy-action@d43c1f16c00cfd3978dde6c07f4bbcf9eb6993ca # @v0.16.1
         with:
-          image-ref: ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:sha-${{ github.event.client_payload.sha }}
+          image-ref: ${{ github.event.client_payload.image }}
           format: "sarif"
           output: "trivy-results.sarif"
           exit-code: "1"

Cargo.toml

@@ -138,3 +138,8 @@ sp-inherents = { version = "4.0.0-dev", git = "https://github.com/subspace/polka
 sp-io = { version = "23.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
 sp-runtime = { version = "24.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
 substrate-prometheus-endpoint = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
+
+# TODO: Importing https://github.com/supranational/blst/pull/203 to take advantage of optimizations introduced there,
+#  switch to upstream once merged or once similar performance improvements land upstream
+[patch."https://github.com/supranational/blst.git"]
+blst = { git = "https://github.com/subspace/blst", rev = "ab042e18cb3b62e131423380513964e4b2c7b445" }

crates/pallet-domains/Cargo.toml

@@ -18,6 +18,7 @@ frame-benchmarking = { version = "4.0.0-dev", default-features = false, git = "h
 frame-support = { version = "4.0.0-dev", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
 frame-system = { version = "4.0.0-dev", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
 log = { version = "0.4.20", default-features = false }
+pallet-balances = { version = "4.0.0-dev", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
 scale-info = { version = "2.7.0", default-features = false, features = ["derive"] }
 sp-consensus-slots = { version = "0.10.0-dev", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
 sp-consensus-subspace = { version = "0.1.0", default-features = false, path = "../sp-consensus-subspace" }
@@ -33,7 +34,7 @@ subspace-runtime-primitives = { version = "0.1.0", default-features = false, pat
 
 [dev-dependencies]
 domain-pallet-executive = { version = "0.1.0", default-features = false, path = "../../domains/pallets/executive" }
-pallet-balances = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
+hex-literal = "0.4.1"
 pallet-timestamp = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
 pallet-block-fees = { version = "0.1.0", default-features = false, path = "../../domains/pallets/block-fees" }
 sp-externalities = { version = "0.19.0", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
@@ -49,6 +50,7 @@ std = [
     "frame-support/std",
     "frame-system/std",
     "log/std",
+    "pallet-balances/std",
     "scale-info/std",
     "sp-consensus-slots/std",
     "sp-consensus-subspace/std",

crates/pallet-domains/src/benchmarking.rs

@@ -225,6 +225,7 @@ mod benchmarks {
             bundle_slot_probability: (1, 1),
             target_bundles_per_block: 10,
             operator_allow_list: OperatorAllowList::Anyone,
+            initial_balances: Default::default(),
         };
 
         #[extrinsic_call]
@@ -434,6 +435,7 @@ mod benchmarks {
             bundle_slot_probability: (1, 1),
             target_bundles_per_block: 10,
             operator_allow_list: OperatorAllowList::Anyone,
+            initial_balances: Default::default(),
         };
 
         assert_ok!(Domains::<T>::instantiate_domain(

crates/pallet-domains/src/block_tree.rs

@@ -10,7 +10,10 @@ use frame_support::{ensure, PalletError};
 use scale_info::TypeInfo;
 use sp_core::Get;
 use sp_domains::merkle_tree::MerkleTree;
-use sp_domains::{ConfirmedDomainBlock, DomainId, ExecutionReceipt, OperatorId};
+use sp_domains::{
+    ChainId, ConfirmedDomainBlock, DomainId, DomainsTransfersTracker, ExecutionReceipt, OperatorId,
+    Transfers,
+};
 use sp_runtime::traits::{BlockNumberProvider, CheckedSub, One, Saturating, Zero};
 use sp_std::cmp::Ordering;
 use sp_std::collections::btree_map::BTreeMap;
@@ -34,6 +37,9 @@ pub enum Error {
     InvalidExecutionTrace,
     UnavailableConsensusBlockHash,
     InvalidStateRoot,
+    BalanceOverflow,
+    DomainTransfersTracking,
+    InvalidDomainTransfers,
 }
 
 #[derive(TypeInfo, Debug, Encode, Decode, Clone, PartialEq, Eq)]
@@ -344,6 +350,21 @@ pub(crate) fn process_execution_receipt<T: Config>(
                     execution_receipt.consensus_block_number,
                 );
 
+                let block_fees = execution_receipt
+                    .block_fees
+                    .total_fees()
+                    .ok_or(Error::BalanceOverflow)?;
+
+                ensure!(
+                    execution_receipt
+                        .transfers
+                        .is_valid(ChainId::Domain(domain_id)),
+                    Error::InvalidDomainTransfers
+                );
+
+                update_domain_transfers::<T>(domain_id, &execution_receipt.transfers, block_fees)
+                    .map_err(|_| Error::DomainTransfersTracking)?;
+
                 LatestConfirmedDomainBlock::<T>::insert(
                     domain_id,
                     ConfirmedDomainBlock {
@@ -380,6 +401,59 @@ pub(crate) fn process_execution_receipt<T: Config>(
     Ok(None)
 }
 
+type TransferTrackerError<T> =
+    <<T as Config>::DomainsTransfersTracker as DomainsTransfersTracker<BalanceOf<T>>>::Error;
+
+/// Updates domain transfers for following scenarios
+/// 1. Block fees are burned on domain
+/// 2. Confirming incoming XDM transfers to the Domain
+/// 3. Noting outgoing transfers from the domain
+/// 4. Cancelling outgoing transfers from the domain.
+fn update_domain_transfers<T: Config>(
+    domain_id: DomainId,
+    transfers: &Transfers<BalanceOf<T>>,
+    block_fees: BalanceOf<T>,
+) -> Result<(), TransferTrackerError<T>> {
+    let Transfers {
+        transfers_in,
+        transfers_out,
+        transfers_rejected,
+        rejected_transfers_claimed,
+    } = transfers;
+
+    // confirm incoming transfers
+    let er_chain_id = ChainId::Domain(domain_id);
+    transfers_in
+        .iter()
+        .try_for_each(|(from_chain_id, amount)| {
+            T::DomainsTransfersTracker::confirm_transfer(*from_chain_id, er_chain_id, *amount)
+        })?;
+
+    // note outgoing transfers
+    transfers_out.iter().try_for_each(|(to_chain_id, amount)| {
+        T::DomainsTransfersTracker::note_transfer(er_chain_id, *to_chain_id, *amount)
+    })?;
+
+    // note rejected transfers
+    transfers_rejected
+        .iter()
+        .try_for_each(|(from_chain_id, amount)| {
+            T::DomainsTransfersTracker::reject_transfer(*from_chain_id, er_chain_id, *amount)
+        })?;
+
+    // claim rejected transfers
+    rejected_transfers_claimed
+        .iter()
+        .try_for_each(|(to_chain_id, amount)| {
+            T::DomainsTransfersTracker::claim_rejected_transfer(er_chain_id, *to_chain_id, *amount)
+        })?;
+
+    // deduct execution fees from domain
+    T::DomainsTransfersTracker::reduce_domain_balance(domain_id, block_fees)?;
+
+    Ok(())
+}
+
 fn add_new_receipt_to_block_tree<T: Config>(
     domain_id: DomainId,
     submitter: OperatorId,
@@ -687,7 +761,7 @@ mod tests {
                 H256::random(),
                 stale_receipt,
             );
-            assert!(crate::Pallet::<Test>::submit_bundle(RawOrigin::None.into(), bundle,).is_err());
+            assert!(crate::Pallet::<Test>::submit_bundle(RawOrigin::None.into(), bundle).is_err());
 
             assert_eq!(
                 BlockTreeNodes::<Test>::get(stale_receipt_hash)
@@ -735,7 +809,7 @@ mod tests {
                 H256::random(),
                 previous_head_receipt,
             );
-            assert!(crate::Pallet::<Test>::submit_bundle(RawOrigin::None.into(), bundle,).is_err());
+            assert!(crate::Pallet::<Test>::submit_bundle(RawOrigin::None.into(), bundle).is_err());
         });
     }