feat(torii): fetch and process erc721 metadata and image

commit-id:274aaa3a

Cargo.toml

@@ -169,14 +169,17 @@ colored_json = "3.2.0"
 console = "0.15.7"
 convert_case = "0.6.0"
 crypto-bigint = { version = "0.5.3", features = [ "serde" ] }
+data-url = "0.3"
 derive_more = "0.99.17"
 flate2 = "1.0.24"
+fluent-uri = "0.3"
 futures = "0.3.30"
 futures-util = "0.3.30"
 hashlink = "0.9.1"
 hex = "0.4.3"
 hex-literal = "0.4.1"
 http = "0.2.9"
+image = "0.25.2"
 indexmap = "2.2.5"
 indoc = "1.0.7"
 itertools = "0.12.1"
@@ -224,6 +227,9 @@ tracing-log = "0.1.3"
 tracing-subscriber = { version = "0.3.16", features = [ "env-filter", "json" ] }
 url = { version = "2.4.0", features = [ "serde" ] }
 walkdir = "2.5.0"
+# TODO: see if we still need the git version
+ipfs-api-backend-hyper = { git = "https://github.com/ferristseng/rust-ipfs-api", rev = "af2c17f7b19ef5b9898f458d97a90055c3605633", features = [ "with-hyper-rustls", "with-send-sync" ] }
+mime_guess = "2.0"
 
 # server
 hyper = "0.14.27"

bin/torii/Cargo.toml

@@ -33,22 +33,22 @@ starknet.workspace = true
 tokio-stream = "0.1.11"
 tokio-util = "0.7.7"
 tokio.workspace = true
+toml.workspace = true
 torii-cli.workspace = true
 torii-core.workspace = true
 torii-graphql.workspace = true
 torii-grpc = { workspace = true, features = [ "server" ] }
 torii-relay.workspace = true
 torii-server.workspace = true
 tower.workspace = true
-toml.workspace = true
 
+clap_config = "0.1.1"
+tempfile.workspace = true
 tower-http.workspace = true
 tracing-subscriber.workspace = true
 tracing.workspace = true
 url.workspace = true
 webbrowser = "0.8"
-tempfile.workspace = true
-clap_config = "0.1.1"
 
 [dev-dependencies]
 assert_matches.workspace = true

bin/torii/src/main.rs

@@ -16,7 +16,10 @@ use std::str::FromStr;
 use std::sync::Arc;
 use std::time::Duration;
 
+use anyhow::Context;
+use camino::Utf8PathBuf;
 use clap::Parser;
+use clap::{ArgAction, Parser};
 use dojo_metrics::exporters::prometheus::PrometheusRecorder;
 use dojo_world::contracts::world::WorldContractReader;
 use sqlx::sqlite::{
@@ -25,7 +28,7 @@ use sqlx::sqlite::{
 use sqlx::SqlitePool;
 use starknet::providers::jsonrpc::HttpTransport;
 use starknet::providers::JsonRpcClient;
-use tempfile::NamedTempFile;
+use tempfile::{NamedTempFile, TempDir};
 use tokio::sync::broadcast;
 use tokio::sync::broadcast::Sender;
 use tokio_stream::StreamExt;
@@ -45,6 +48,111 @@ use url::{form_urlencoded, Url};
 
 pub(crate) const LOG_TARGET: &str = "torii::cli";
 
+/// Dojo World Indexer
+#[derive(Parser, Debug)]
+#[command(name = "torii", author, version, about, long_about = None)]
+struct Args {
+    /// The world to index
+    #[arg(short, long = "world", env = "DOJO_WORLD_ADDRESS")]
+    world_address: Option<Felt>,
+
+    /// The sequencer rpc endpoint to index.
+    #[arg(long, value_name = "URL", default_value = ":5050", value_parser = parse_url)]
+    rpc: Url,
+
+    /// Database filepath (ex: indexer.db). If specified file doesn't exist, it will be
+    /// created. Defaults to in-memory database
+    #[arg(short, long, default_value = "")]
+    database: String,
+
+    /// Address to serve api endpoints at.
+    #[arg(long, value_name = "SOCKET", default_value = "0.0.0.0:8080", value_parser = parse_socket_address)]
+    addr: SocketAddr,
+
+    /// Port to serve Libp2p TCP & UDP Quic transports
+    #[arg(long, value_name = "PORT", default_value = "9090")]
+    relay_port: u16,
+
+    /// Port to serve Libp2p WebRTC transport
+    #[arg(long, value_name = "PORT", default_value = "9091")]
+    relay_webrtc_port: u16,
+
+    /// Port to serve Libp2p WebRTC transport
+    #[arg(long, value_name = "PORT", default_value = "9092")]
+    relay_websocket_port: u16,
+
+    /// Path to a local identity key file. If not specified, a new identity will be generated
+    #[arg(long, value_name = "PATH")]
+    relay_local_key_path: Option<String>,
+
+    /// Path to a local certificate file. If not specified, a new certificate will be generated
+    /// for WebRTC connections
+    #[arg(long, value_name = "PATH")]
+    relay_cert_path: Option<String>,
+
+    /// Specify allowed origins for api endpoints (comma-separated list of allowed origins, or "*"
+    /// for all)
+    #[arg(long)]
+    #[arg(value_delimiter = ',')]
+    allowed_origins: Option<Vec<String>>,
+
+    /// The external url of the server, used for configuring the GraphQL Playground in a hosted
+    /// environment
+    #[arg(long, value_parser = parse_url)]
+    external_url: Option<Url>,
+
+    /// Enable Prometheus metrics.
+    ///
+    /// The metrics will be served at the given interface and port.
+    #[arg(long, value_name = "SOCKET", value_parser = parse_socket_address, help_heading = "Metrics")]
+    metrics: Option<SocketAddr>,
+
+    /// Open World Explorer on the browser.
+    #[arg(long)]
+    explorer: bool,
+
+    /// Chunk size of the events page when indexing using events
+    #[arg(long, default_value = "1024")]
+    events_chunk_size: u64,
+
+    /// Number of blocks to process before commiting to DB
+    #[arg(long, default_value = "10240")]
+    blocks_chunk_size: u64,
+
+    /// Enable indexing pending blocks
+    #[arg(long, action = ArgAction::Set, default_value_t = true)]
+    index_pending: bool,
+
+    /// Polling interval in ms
+    #[arg(long, default_value = "500")]
+    polling_interval: u64,
+
+    /// Max concurrent tasks
+    #[arg(long, default_value = "100")]
+    max_concurrent_tasks: usize,
+
+    /// Whether or not to index world transactions
+    #[arg(long, action = ArgAction::Set, default_value_t = false)]
+    index_transactions: bool,
+
+    /// Whether or not to index raw events
+    #[arg(long, action = ArgAction::Set, default_value_t = true)]
+    index_raw_events: bool,
+
+    /// ERC contract addresses to index
+    #[arg(long, value_parser = parse_erc_contracts)]
+    #[arg(conflicts_with = "config")]
+    contracts: Option<std::vec::Vec<Contract>>,
+
+    /// Configuration file
+    #[arg(long)]
+    config: Option<PathBuf>,
+
+    /// Path to a directory to store ERC artifacts
+    #[arg(long)]
+    artifacts_path: Option<Utf8PathBuf>,
+}
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     let mut args = ToriiArgs::parse().with_config_file()?;
@@ -109,7 +217,21 @@ async fn main() -> anyhow::Result<()> {
     // Get world address
     let world = WorldContractReader::new(world_address, provider.clone());
 
-    let (mut executor, sender) = Executor::new(pool.clone(), shutdown_tx.clone()).await?;
+    // let (mut executor, sender) = Executor::new(pool.clone(), shutdown_tx.clone()).await?;
+    let contracts = args
+        .indexing
+        .contracts
+        .iter()
+        .map(|contract| (contract.address, contract.r#type))
+        .collect();
+
+    let (mut executor, sender) = Executor::new(
+        pool.clone(),
+        shutdown_tx.clone(),
+        provider.clone(),
+        args.max_concurrent_tasks,
+    )
+    .await?;
     tokio::spawn(async move {
         executor.run().await.unwrap();
     });
@@ -184,6 +306,7 @@ async fn main() -> anyhow::Result<()> {
         args.server.http_cors_origins.filter(|cors_origins| !cors_origins.is_empty()),
         Some(grpc_addr),
         None,
+        Some(artifacts_addr),
     ));
 
     let graphql_server = spawn_rebuilding_graphql_server(
@@ -201,6 +324,7 @@ async fn main() -> anyhow::Result<()> {
     info!(target: LOG_TARGET, endpoint = %addr, "Starting torii endpoint.");
     info!(target: LOG_TARGET, endpoint = %gql_endpoint, "Serving Graphql playground.");
     info!(target: LOG_TARGET, url = %explorer_url, "Serving World Explorer.");
+    info!(target: LOG_TARGET, path = %artifacts_path, "Serving ERC artifacts at path");
 
     if args.explorer {
         if let Err(e) = webbrowser::open(&explorer_url) {
@@ -222,13 +346,15 @@ async fn main() -> anyhow::Result<()> {
     let graphql_server_handle = tokio::spawn(graphql_server);
     let grpc_server_handle = tokio::spawn(grpc_server);
     let libp2p_relay_server_handle = tokio::spawn(async move { libp2p_relay_server.run().await });
+    let artifacts_server_handle = tokio::spawn(artifacts_server);
 
     tokio::select! {
         res = engine_handle => res??,
         res = proxy_server_handle => res??,
         res = graphql_server_handle => res?,
         res = grpc_server_handle => res??,
         res = libp2p_relay_server_handle => res?,
+        res = artifacts_server_handle => res?,
         _ = dojo_utils::signal::wait_signals() => {},
     };
 

crates/dojo-world/Cargo.toml

@@ -0,0 +1,55 @@
+[package]
+description = "Dojo world specification. For example, crates and flags used for compilation."
+edition.workspace = true
+license-file.workspace = true
+name = "dojo-world"
+repository.workspace = true
+version.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+async-trait.workspace = true
+cairo-lang-filesystem.workspace = true
+cairo-lang-project.workspace = true
+cairo-lang-starknet-classes.workspace = true
+cairo-lang-starknet.workspace = true
+camino.workspace = true
+convert_case.workspace = true
+dojo-utils = { workspace = true, optional = true }
+num-traits = { workspace = true, optional = true }
+regex.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+serde_with.workspace = true
+smol_str.workspace = true
+starknet-crypto.workspace = true
+starknet.workspace = true
+thiserror.workspace = true
+topological-sort.workspace = true
+tracing.workspace = true
+
+cainome.workspace = true
+dojo-types = { path = "../dojo-types", optional = true }
+http = { workspace = true, optional = true }
+ipfs-api-backend-hyper = { workspace = true, optional = true }
+scarb = { workspace = true, optional = true }
+tokio = { version = "1.32.0", features = [ "time" ], default-features = false, optional = true }
+toml.workspace = true
+url = { workspace = true, optional = true }
+walkdir.workspace = true
+
+[dev-dependencies]
+assert_fs.workspace = true
+assert_matches.workspace = true
+dojo-lang.workspace = true
+dojo-test-utils = { path = "../dojo-test-utils" }
+katana-runner.workspace = true
+similar-asserts.workspace = true
+tempfile.workspace = true
+tokio.workspace = true
+
+[features]
+contracts = [ "dep:dojo-types", "dep:http", "dep:num-traits" ]
+manifest = [ "contracts", "dep:dojo-types", "dep:scarb", "dep:url" ]
+metadata = [ "dep:ipfs-api-backend-hyper", "dep:scarb", "dep:url" ]
+migration = [ "dep:dojo-utils", "dep:scarb", "dep:tokio", "manifest" ]

crates/sozo/ops/Cargo.toml

@@ -11,8 +11,8 @@ async-trait.workspace = true
 cainome.workspace = true
 colored.workspace = true
 colored_json.workspace = true
-dojo-utils.workspace = true
 dojo-types.workspace = true
+dojo-utils.workspace = true
 dojo-world.workspace = true
 futures.workspace = true
 num-traits.workspace = true
@@ -21,8 +21,8 @@ serde_json.workspace = true
 serde_with.workspace = true
 sozo-walnut = { workspace = true, optional = true }
 spinoff.workspace = true
-starknet.workspace = true
 starknet-crypto.workspace = true
+starknet.workspace = true
 thiserror.workspace = true
 toml.workspace = true
 tracing.workspace = true
@@ -33,11 +33,11 @@ katana-runner = { workspace = true, optional = true }
 [dev-dependencies]
 assert_fs.workspace = true
 dojo-test-utils = { workspace = true, features = [ "build-examples" ] }
-ipfs-api-backend-hyper = { git = "https://github.com/ferristseng/rust-ipfs-api", rev = "af2c17f7b19ef5b9898f458d97a90055c3605633", features = [ "with-hyper-rustls" ] }
+ipfs-api-backend-hyper.workspace = true
 katana-runner.workspace = true
-tokio.workspace = true
 scarb.workspace = true
 sozo-scarbext.workspace = true
+tokio.workspace = true
 
 [features]
 test-utils = [ "dep:dojo-test-utils", "dep:katana-runner" ]

crates/torii/core/Cargo.toml

@@ -16,6 +16,7 @@ bitflags = "2.6.0"
 cainome.workspace = true
 chrono.workspace = true
 crypto-bigint.workspace = true
+data-url.workspace = true
 dojo-types.workspace = true
 dojo-world.workspace = true
 futures-channel = "0.3.0"
@@ -31,8 +32,9 @@ sqlx.workspace = true
 starknet-crypto.workspace = true
 starknet.workspace = true
 thiserror.workspace = true
-tokio = { version = "1.32.0", features = [ "sync", "macros" ], default-features = true }
+tokio = { version = "1.32.0", features = [ "macros", "sync" ], default-features = true }
 # tokio-stream = "0.1.11"
+ipfs-api-backend-hyper.workspace = true
 tokio-util.workspace = true
 tracing.workspace = true
 
@@ -41,5 +43,5 @@ dojo-test-utils.workspace = true
 dojo-utils.workspace = true
 katana-runner.workspace = true
 scarb.workspace = true
-tempfile.workspace = true
 sozo-scarbext.workspace = true
+tempfile.workspace = true

crates/torii/core/src/engine.rs

@@ -277,8 +277,9 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
 
                             match self.process(fetch_result).await {
                                 Ok(_) => {
-                                    self.db.execute().await?;
+                                    self.db.flush().await?;
                                     self.db.apply_cache_diff().await?;
+                                    self.db.execute().await?;
                                 },
                                 Err(e) => {
                                     error!(target: LOG_TARGET, error = %e, "Processing fetched data.");
@@ -646,8 +647,7 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
 
             unique_contracts.insert(event.from_address);
 
-            Self::process_event(
-                self,
+            self.process_event(
                 block_number,
                 block_timestamp,
                 &event_id,
@@ -707,8 +707,7 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
                 let event_id =
                     format!("{:#064x}:{:#x}:{:#04x}", block_number, *transaction_hash, event_idx);
 
-                Self::process_event(
-                    self,
+                self.process_event(
                     block_number,
                     block_timestamp,
                     &event_id,
@@ -720,8 +719,7 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
             }
 
             if self.config.flags.contains(IndexingFlags::TRANSACTIONS) {
-                Self::process_transaction(
-                    self,
+                self.process_transaction(
                     block_number,
                     block_timestamp,
                     *transaction_hash,