ENG-578: IT (Part 5) - Docker start

.github/workflows/fendermint-publish.yaml

@@ -80,10 +80,6 @@ jobs:
 
           echo "IMAGE_TAG=$IMAGE_ID:$VERSION" >> $GITHUB_OUTPUT
 
-      # Pull the latest docker image. Its layers might be reused during the build.
-      - name: Docker Pull
-        run: docker pull ghcr.io/consensus-shipyard/fendermint:latest
-
       - name: Docker Deps
         run: |
           cd fendermint && make docker-deps

.github/workflows/fendermint-test.yaml

@@ -64,6 +64,18 @@ jobs:
       RUSTFLAGS: -Dwarnings
 
     steps:
+      # https://github.com/marketplace/actions/free-disk-space-ubuntu
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          large-packages: false
+          swap-storage: false
+          docker-images: false
+          android: true
+          dotnet: true
+          haskell: true
+
       - name: Check out the project
         uses: actions/checkout@v3
         with:
@@ -90,11 +102,7 @@ jobs:
           path: ./contracts/out
           key: contracts-abi-${{ hashFiles('./contracts/src/**/*.sol') }}
 
-      # Pull the latest docker image. Its layers might be reused during the build.
-      - name: Docker Pull
-        run: docker pull ghcr.io/consensus-shipyard/fendermint:latest
-
       - name: ${{ matrix.make.name }}
         env:
           PROMTAIL_CLIENT_URL: ${{ secrets.PROMTAIL_CLIENT_URL }}
-        run: cd fendermint && make ${{ matrix.make.task }}
+        run: cd fendermint && make ${{ matrix.make.task }}

fendermint/.gitignore

@@ -4,6 +4,7 @@ docker/.artifacts
 docker/Dockerfile
 cometbft
 test-network
+testing/materializer/tests/docker-materializer-data
 .idea
 .make
 .contracts-gen

fendermint/Makefile

@@ -38,21 +38,25 @@ install: $(IPC_ACTORS_GEN)
 	cargo install --locked --path app
 
 # Using --release for testing because wasm can otherwise be slow.
+# Excluding the materializer tests because they rely on docker; we could test with --lib
 test: $(BUILTIN_ACTORS_BUNDLE) $(IPC_ACTORS_GEN)
 	FM_BUILTIN_ACTORS_BUNDLE=$(BUILTIN_ACTORS_BUNDLE) \
 	FM_CUSTOM_ACTORS_BUNDLE=$(CUSTOM_ACTORS_BUNDLE) \
 	FM_CONTRACTS_DIR=$(IPC_ACTORS_OUT) \
-	cargo test --release $(PACKAGE)
+	cargo test --release $(shell echo $(PACKAGE) | sed 's/--package fendermint_materializer//g')
 
+# Not using --release beause the build has been done in docker and the wasm code runs inside the container.
 e2e: docker-build | cargo-make
 	cd testing/smoke-test    && cargo make --profile $(PROFILE)
 	cd testing/snapshot-test && cargo make --profile $(PROFILE)
 	cd testing/graph-test    && cargo make --profile $(PROFILE)
+	PROFILE=$(PROFILE) cargo test --package fendermint_materializer
 
 clean:
 	cargo clean
 	rm $(BUILTIN_ACTORS_BUNDLE)
 	rm $(CUSTOM_ACTORS_BUNDLE)
+	rm -rf testing/materializer/tests/docker-materializer
 
 lint: \
 	check-fmt \

fendermint/abci/Cargo.toml

@@ -10,6 +10,7 @@ license.workspace = true
 async-trait = { workspace = true }
 futures = { workspace = true }
 tower = "0.4"
+tracing = { workspace = true }
 
 tower-abci = { workspace = true }
 tendermint = { workspace = true }
@@ -20,5 +21,4 @@ async-stm = { workspace = true }
 im = { workspace = true }
 structopt = "0.3"
 tokio = { workspace = true }
-tracing = { workspace = true }
 tracing-subscriber = { workspace = true }

fendermint/abci/src/application.rs

@@ -162,28 +162,34 @@ where
 
         let res = async move {
             let res = match req {
-                Request::Echo(r) => Response::Echo(app.echo(r).await?),
-                Request::Info(r) => Response::Info(app.info(r).await?),
-                Request::InitChain(r) => Response::InitChain(app.init_chain(r).await?),
-                Request::Query(r) => Response::Query(app.query(r).await?),
-                Request::CheckTx(r) => Response::CheckTx(app.check_tx(r).await?),
+                Request::Echo(r) => Response::Echo(log_error(app.echo(r).await)?),
+                Request::Info(r) => Response::Info(log_error(app.info(r).await)?),
+                Request::InitChain(r) => Response::InitChain(log_error(app.init_chain(r).await)?),
+                Request::Query(r) => Response::Query(log_error(app.query(r).await)?),
+                Request::CheckTx(r) => Response::CheckTx(log_error(app.check_tx(r).await)?),
                 Request::PrepareProposal(r) => {
-                    Response::PrepareProposal(app.prepare_proposal(r).await?)
+                    Response::PrepareProposal(log_error(app.prepare_proposal(r).await)?)
                 }
                 Request::ProcessProposal(r) => {
-                    Response::ProcessProposal(app.process_proposal(r).await?)
+                    Response::ProcessProposal(log_error(app.process_proposal(r).await)?)
+                }
+                Request::BeginBlock(r) => {
+                    Response::BeginBlock(log_error(app.begin_block(r).await)?)
+                }
+                Request::DeliverTx(r) => Response::DeliverTx(log_error(app.deliver_tx(r).await)?),
+                Request::EndBlock(r) => Response::EndBlock(log_error(app.end_block(r).await)?),
+                Request::Commit => Response::Commit(log_error(app.commit().await)?),
+                Request::ListSnapshots => {
+                    Response::ListSnapshots(log_error(app.list_snapshots().await)?)
+                }
+                Request::OfferSnapshot(r) => {
+                    Response::OfferSnapshot(log_error(app.offer_snapshot(r).await)?)
                 }
-                Request::BeginBlock(r) => Response::BeginBlock(app.begin_block(r).await?),
-                Request::DeliverTx(r) => Response::DeliverTx(app.deliver_tx(r).await?),
-                Request::EndBlock(r) => Response::EndBlock(app.end_block(r).await?),
-                Request::Commit => Response::Commit(app.commit().await?),
-                Request::ListSnapshots => Response::ListSnapshots(app.list_snapshots().await?),
-                Request::OfferSnapshot(r) => Response::OfferSnapshot(app.offer_snapshot(r).await?),
                 Request::LoadSnapshotChunk(r) => {
-                    Response::LoadSnapshotChunk(app.load_snapshot_chunk(r).await?)
+                    Response::LoadSnapshotChunk(log_error(app.load_snapshot_chunk(r).await)?)
                 }
                 Request::ApplySnapshotChunk(r) => {
-                    Response::ApplySnapshotChunk(app.apply_snapshot_chunk(r).await?)
+                    Response::ApplySnapshotChunk(log_error(app.apply_snapshot_chunk(r).await)?)
                 }
                 Request::Flush => panic!("Flush should be handled by the Server!"),
             };
@@ -192,3 +198,10 @@ where
         res.boxed()
     }
 }
+
+fn log_error<T>(res: AbciResult<T>) -> AbciResult<T> {
+    if let Err(ref e) = res {
+        tracing::error!("failed to execute ABCI request: {e:#}");
+    }
+    res
+}

fendermint/docker/runner.Dockerfile

@@ -15,6 +15,7 @@ ENV HOME=$FM_HOME_DIR
 WORKDIR $FM_HOME_DIR
 
 EXPOSE 26658
+EXPOSE 8445
 
 ENTRYPOINT ["docker-entry.sh"]
 CMD ["run"]

fendermint/eth/api/examples/ethers.rs

@@ -81,11 +81,11 @@ pub struct Options {
     /// Secret key used to send funds, expected to be in Base64 format.
     ///
     /// Assumed to exist with a non-zero balance.
-    #[arg(long, short)]
+    #[arg(long)]
     pub secret_key_from: PathBuf,
 
     /// Secret key used to receive funds, expected to be in Base64 format.
-    #[arg(long, short)]
+    #[arg(long)]
     pub secret_key_to: PathBuf,
 
     /// Enable DEBUG logs.

fendermint/testing/graph-test/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "graph-test"
+name = "fendermint_graph_test"
 version = "0.1.0"
 edition = "2021"
 

fendermint/testing/graph-test/Makefile.toml

@@ -30,7 +30,7 @@ run_task = { name = [
 [tasks.greeter-example]
 script = """
 cd ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/fendermint
-cargo run -p fendermint_eth_api --release --example greeter -- \
+cargo run -p fendermint_eth_api --example greeter -- \
   --secret-key testing/graph-test/test-data/keys/veronica.sk \
   --out ${TEST_DATA_DIR}/greeter.json \
   ${VERBOSITY}

fendermint/testing/materializer/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "fendermint_testing_materializer"
+name = "fendermint_materializer"
 description = "Toolkit to instantiate testnets"
 version = "0.1.0"
 authors.workspace = true
@@ -14,8 +14,10 @@ async-recursion = { workspace = true }
 async-trait = { workspace = true }
 bollard = { workspace = true }
 ethers = { workspace = true }
+futures = { workspace = true }
 fvm_shared = { workspace = true }
 hex = { workspace = true }
+lazy_static = { workspace = true }
 multihash = { workspace = true }
 serde = { workspace = true }
 serde_with = { workspace = true }
@@ -25,7 +27,6 @@ tendermint-rpc = { workspace = true }
 tokio = { workspace = true }
 tracing = { workspace = true }
 
-lazy_static = { workspace = true, optional = true }
 arbitrary = { workspace = true, optional = true }
 quickcheck = { workspace = true, optional = true }
 
@@ -47,20 +48,20 @@ quickcheck_async = { workspace = true }
 quickcheck_macros = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }
+serial_test = { workspace = true }
 tempfile = { workspace = true }
 tokio = { workspace = true }
 toml = { workspace = true }
 
 # Enable arb on self for tests.
-fendermint_testing_materializer = { path = ".", features = ["arb"] }
+fendermint_materializer = { path = ".", features = ["arb"] }
 fendermint_testing = { path = "..", features = ["golden"] }
 
 [features]
 default = []
 arb = [
   "arbitrary",
   "quickcheck",
-  "lazy_static",
   "fvm_shared/arb",
   "fendermint_testing/arb",
   "fendermint_vm_genesis/arb",