From d50e51b75ba4ae4461178447232423cd9ffcc6e8 Mon Sep 17 00:00:00 2001
From: lambda-0x <0xlambda@protonmail.com>
Date: Sat, 12 Oct 2024 22:00:38 +0530
Subject: [PATCH 1/9] feat(torii/graphql): use `Connection` abstraction to
return data for ercBalance and ercTransfer query
commit-id:bc10539f
---
crates/torii/graphql/src/constants.rs | 2 +
.../graphql/src/object/erc/erc_balance.rs | 201 ++++++++++++++--
.../graphql/src/object/erc/erc_transfer.rs | 220 ++++++++++++++----
crates/torii/graphql/src/object/erc/mod.rs | 14 ++
4 files changed, 372 insertions(+), 65 deletions(-)
diff --git a/crates/torii/graphql/src/constants.rs b/crates/torii/graphql/src/constants.rs
index 2d851f07b1..fc5a376f56 100644
--- a/crates/torii/graphql/src/constants.rs
+++ b/crates/torii/graphql/src/constants.rs
@@ -9,6 +9,8 @@ pub const EVENT_MESSAGE_TABLE: &str = "event_messages";
pub const MODEL_TABLE: &str = "models";
pub const TRANSACTION_TABLE: &str = "transactions";
pub const METADATA_TABLE: &str = "metadata";
+pub const ERC_BALANCE_TABLE: &str = "balances";
+pub const ERC_TRANSFER_TABLE: &str = "erc_transfers";
pub const ID_COLUMN: &str = "id";
pub const EVENT_ID_COLUMN: &str = "event_id";
diff --git a/crates/torii/graphql/src/object/erc/erc_balance.rs b/crates/torii/graphql/src/object/erc/erc_balance.rs
index a749350fee..848303e71a 100644
--- a/crates/torii/graphql/src/object/erc/erc_balance.rs
+++ b/crates/torii/graphql/src/object/erc/erc_balance.rs
@@ -1,15 +1,27 @@
+use async_graphql::connection::PageInfo;
use async_graphql::dynamic::{Field, FieldFuture, InputValue, TypeRef};
use async_graphql::{Name, Value};
use convert_case::{Case, Casing};
use serde::Deserialize;
-use sqlx::{FromRow, Pool, Sqlite, SqliteConnection};
+use sqlx::sqlite::SqliteRow;
+use sqlx::{FromRow, Pool, Row, Sqlite, SqliteConnection};
use starknet_crypto::Felt;
use torii_core::sql::utils::felt_to_sql_string;
use tracing::warn;
-use crate::constants::{ERC_BALANCE_NAME, ERC_BALANCE_TYPE_NAME};
+use super::handle_cursor;
+use crate::constants::{
+ DEFAULT_LIMIT, ERC_BALANCE_NAME, ERC_BALANCE_TABLE, ERC_BALANCE_TYPE_NAME, ID_COLUMN,
+};
use crate::mapping::ERC_BALANCE_TYPE_MAPPING;
+use crate::object::connection::page_info::PageInfoObject;
+use crate::object::connection::{
+ connection_arguments, cursor, parse_connection_arguments, ConnectionArguments,
+};
use crate::object::{BasicObject, ResolvableObject};
+use crate::query::data::count_rows;
+use crate::query::filter::{Comparator, Filter, FilterValue};
+use crate::query::order::{CursorDirection, Direction};
use crate::types::{TypeMapping, ValueMapping};
use crate::utils::extract;
@@ -38,20 +50,39 @@ impl ResolvableObject for ErcBalanceObject {
TypeRef::named_nn(TypeRef::STRING),
);
- let field = Field::new(self.name().0, TypeRef::named_list(self.type_name()), move |ctx| {
- FieldFuture::new(async move {
- let mut conn = ctx.data::<Pool<Sqlite>>()?.acquire().await?;
- let address = extract::<Felt>(
- ctx.args.as_index_map(),
- &account_address.to_case(Case::Camel),
- )?;
+ let mut field = Field::new(
+ self.name().0,
+ TypeRef::named(format!("{}Connection", self.type_name())),
+ move |ctx| {
+ FieldFuture::new(async move {
+ let mut conn = ctx.data::<Pool<Sqlite>>()?.acquire().await?;
+ let connection = parse_connection_arguments(&ctx)?;
+ let address = extract::<Felt>(
+ ctx.args.as_index_map(),
+ &account_address.to_case(Case::Camel),
+ )?;
- let erc_balances = fetch_erc_balances(&mut conn, address).await?;
+ let filter = vec![Filter {
+ field: "account_address".to_string(),
+ comparator: Comparator::Eq,
+ value: FilterValue::String(felt_to_sql_string(&address)),
+ }];
- Ok(Some(Value::List(erc_balances)))
- })
- })
+ let total_count =
+ count_rows(&mut conn, ERC_BALANCE_TABLE, &None, &Some(filter)).await?;
+
+ let (data, page_info) =
+ fetch_erc_balances(&mut conn, address, &connection, total_count).await?;
+
+ let results = erc_balance_connection_output(&data, total_count, page_info)?;
+
+ Ok(Some(Value::Object(results)))
+ })
+ },
+ )
.argument(argument);
+
+ field = connection_arguments(field);
vec![field]
}
}
@@ -59,20 +90,132 @@ impl ResolvableObject for ErcBalanceObject {
async fn fetch_erc_balances(
conn: &mut SqliteConnection,
address: Felt,
-) -> sqlx::Result<Vec<Value>> {
- let query = "SELECT t.contract_address, t.name, t.symbol, t.decimals, b.balance, b.token_id, \
- c.contract_type
- FROM balances b
+ connection: &ConnectionArguments,
+ total_count: i64,
+) -> sqlx::Result<(Vec<SqliteRow>, PageInfo)> {
+ let table_name = ERC_BALANCE_TABLE;
+ let id_column = format!("b.{}", ID_COLUMN);
+
+ let mut query = format!(
+ "SELECT b.id, t.contract_address, t.name, t.symbol, t.decimals, b.balance, b.token_id, \
+ c.contract_type
+ FROM {table_name} b
JOIN tokens t ON b.token_id = t.id
- JOIN contracts c ON t.contract_address = c.contract_address
- WHERE b.account_address = ?";
+ JOIN contracts c ON t.contract_address = c.contract_address"
+ );
+ let mut conditions = vec!["b.account_address = ?".to_string()];
+
+ let mut cursor_param = &connection.after;
+ if let Some(after_cursor) = &connection.after {
+ conditions.push(handle_cursor(after_cursor, CursorDirection::After, ID_COLUMN)?);
+ }
+
+ if let Some(before_cursor) = &connection.before {
+ cursor_param = &connection.before;
+ conditions.push(handle_cursor(before_cursor, CursorDirection::Before, ID_COLUMN)?);
+ }
+
+ if !conditions.is_empty() {
+ query.push_str(&format!(" WHERE {}", conditions.join(" AND ")));
+ }
+
+ let is_cursor_based = connection.first.or(connection.last).is_some() || cursor_param.is_some();
+
+ let data_limit =
+ connection.first.or(connection.last).or(connection.limit).unwrap_or(DEFAULT_LIMIT);
+ let limit = if is_cursor_based {
+ match &cursor_param {
+ Some(_) => data_limit + 2,
+ None => data_limit + 1, // prev page does not exist
+ }
+ } else {
+ data_limit
+ };
- let rows = sqlx::query(query).bind(felt_to_sql_string(&address)).fetch_all(conn).await?;
+ let order_direction = match (connection.first, connection.last) {
+ (Some(_), _) => Direction::Desc,
+ (_, Some(_)) => Direction::Asc,
+ _ => Direction::Desc,
+ };
- let mut erc_balances = Vec::new();
+ query.push_str(&format!(" ORDER BY {id_column} {} LIMIT {limit}", order_direction.as_ref()));
+
+ if let Some(offset) = connection.offset {
+ query.push_str(&format!(" OFFSET {}", offset));
+ }
+
+ let mut data = sqlx::query(&query).bind(felt_to_sql_string(&address)).fetch_all(conn).await?;
+ let mut page_info = PageInfo {
+ has_previous_page: false,
+ has_next_page: false,
+ start_cursor: None,
+ end_cursor: None,
+ };
+
+ if data.is_empty() {
+ Ok((data, page_info))
+ } else if is_cursor_based {
+ match cursor_param {
+ Some(cursor_query) => {
+ let first_cursor = cursor::encode(
+ &data[0].try_get::<String, &str>(&id_column)?,
+ &data[0].try_get_unchecked::<String, &str>(&id_column)?,
+ );
+
+ if &first_cursor == cursor_query && data.len() != 1 {
+ data.remove(0);
+ page_info.has_previous_page = true;
+ } else {
+ data.pop();
+ }
+
+ if data.len() as u64 == limit - 1 {
+ page_info.has_next_page = true;
+ data.pop();
+ }
+ }
+ None => {
+ if data.len() as u64 == limit {
+ page_info.has_next_page = true;
+ data.pop();
+ }
+ }
+ }
+
+ if !data.is_empty() {
+ page_info.start_cursor = Some(cursor::encode(
+ &data[0].try_get::<String, &str>(ID_COLUMN)?,
+ &data[0].try_get_unchecked::<String, &str>(ID_COLUMN)?,
+ ));
+ page_info.end_cursor = Some(cursor::encode(
+ &data[data.len() - 1].try_get::<String, &str>(ID_COLUMN)?,
+ &data[data.len() - 1].try_get_unchecked::<String, &str>(ID_COLUMN)?,
+ ));
+ }
+
+ Ok((data, page_info))
+ } else {
+ let offset = connection.offset.unwrap_or(0);
+ if 1 < offset && offset < total_count as u64 {
+ page_info.has_previous_page = true;
+ }
+ if limit + offset < total_count as u64 {
+ page_info.has_next_page = true;
+ }
+
+ Ok((data, page_info))
+ }
+}
- for row in rows {
- let row = BalanceQueryResultRaw::from_row(&row)?;
+fn erc_balance_connection_output(
+ data: &[SqliteRow],
+ total_count: i64,
+ page_info: PageInfo,
+) -> sqlx::Result<ValueMapping> {
+ let mut edges = Vec::new();
+ for row in data {
+ let row = BalanceQueryResultRaw::from_row(row)?;
+ let cursor = cursor::encode(&row.id, &row.id);
let balance_value = match row.contract_type.to_lowercase().as_str() {
"erc20" => {
@@ -116,10 +259,17 @@ async fn fetch_erc_balances(
}
};
- erc_balances.push(balance_value);
+ edges.push(Value::Object(ValueMapping::from([
+ (Name::new("node"), balance_value),
+ (Name::new("cursor"), Value::String(cursor)),
+ ])));
}
- Ok(erc_balances)
+ Ok(ValueMapping::from([
+ (Name::new("totalCount"), Value::from(total_count)),
+ (Name::new("edges"), Value::List(edges)),
+ (Name::new("pageInfo"), PageInfoObject::value(page_info)),
+ ]))
}
// TODO: This would be required when subscriptions are needed
@@ -133,6 +283,7 @@ async fn fetch_erc_balances(
#[derive(FromRow, Deserialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
struct BalanceQueryResultRaw {
+ pub id: String,
pub contract_address: String,
pub name: String,
pub symbol: String,
diff --git a/crates/torii/graphql/src/object/erc/erc_transfer.rs b/crates/torii/graphql/src/object/erc/erc_transfer.rs
index ee522fef20..e1c950db39 100644
--- a/crates/torii/graphql/src/object/erc/erc_transfer.rs
+++ b/crates/torii/graphql/src/object/erc/erc_transfer.rs
@@ -1,16 +1,26 @@
+use async_graphql::connection::PageInfo;
use async_graphql::dynamic::{Field, FieldFuture, InputValue, TypeRef};
use async_graphql::{Name, Value};
use convert_case::{Case, Casing};
use serde::Deserialize;
-use sqlx::{FromRow, Pool, Sqlite, SqliteConnection};
+use sqlx::sqlite::SqliteRow;
+use sqlx::{FromRow, Pool, Row, Sqlite, SqliteConnection};
use starknet_crypto::Felt;
use torii_core::engine::get_transaction_hash_from_event_id;
use torii_core::sql::utils::felt_to_sql_string;
use tracing::warn;
-use crate::constants::{ERC_TRANSFER_NAME, ERC_TRANSFER_TYPE_NAME};
+use super::handle_cursor;
+use crate::constants::{
+ DEFAULT_LIMIT, ERC_TRANSFER_NAME, ERC_TRANSFER_TABLE, ERC_TRANSFER_TYPE_NAME, ID_COLUMN,
+};
use crate::mapping::ERC_TRANSFER_TYPE_MAPPING;
+use crate::object::connection::page_info::PageInfoObject;
+use crate::object::connection::{
+ connection_arguments, cursor, parse_connection_arguments, ConnectionArguments,
+};
use crate::object::{BasicObject, ResolvableObject};
+use crate::query::order::{CursorDirection, Direction};
use crate::types::{TypeMapping, ValueMapping};
use crate::utils::extract;
@@ -34,31 +44,44 @@ impl BasicObject for ErcTransferObject {
impl ResolvableObject for ErcTransferObject {
fn resolvers(&self) -> Vec<Field> {
let account_address = "account_address";
- let limit = "limit";
let arg_addr = InputValue::new(
account_address.to_case(Case::Camel),
TypeRef::named_nn(TypeRef::STRING),
);
- let arg_limit =
- InputValue::new(limit.to_case(Case::Camel), TypeRef::named_nn(TypeRef::INT));
-
- let field = Field::new(self.name().0, TypeRef::named_list(self.type_name()), move |ctx| {
- FieldFuture::new(async move {
- let mut conn = ctx.data::<Pool<Sqlite>>()?.acquire().await?;
- let address = extract::<Felt>(
- ctx.args.as_index_map(),
- &account_address.to_case(Case::Camel),
- )?;
- let limit = extract::<u64>(ctx.args.as_index_map(), &limit.to_case(Case::Camel))?;
- let limit: u32 = limit.try_into()?;
-
- let erc_transfers = fetch_erc_transfers(&mut conn, address, limit).await?;
-
- Ok(Some(Value::List(erc_transfers)))
- })
- })
- .argument(arg_addr)
- .argument(arg_limit);
+
+ let mut field = Field::new(
+ self.name().0,
+ TypeRef::named(format!("{}Connection", self.type_name())),
+ move |ctx| {
+ FieldFuture::new(async move {
+ let mut conn = ctx.data::<Pool<Sqlite>>()?.acquire().await?;
+ let connection = parse_connection_arguments(&ctx)?;
+ let address = extract::<Felt>(
+ ctx.args.as_index_map(),
+ &account_address.to_case(Case::Camel),
+ )?;
+
+ let total_count: (i64,) = sqlx::query_as(
+ "SELECT COUNT(*) FROM erc_transfers WHERE from_address = ? OR to_address \
+ = ?",
+ )
+ .bind(felt_to_sql_string(&address))
+ .bind(felt_to_sql_string(&address))
+ .fetch_one(&mut *conn)
+ .await?;
+ let total_count = total_count.0;
+
+ let (data, page_info) =
+ fetch_erc_transfers(&mut conn, address, &connection, total_count).await?;
+ let results = erc_transfer_connection_output(&data, total_count, page_info)?;
+
+ Ok(Some(Value::Object(results)))
+ })
+ },
+ )
+ .argument(arg_addr);
+
+ field = connection_arguments(field);
vec![field]
}
}
@@ -66,9 +89,13 @@ impl ResolvableObject for ErcTransferObject {
async fn fetch_erc_transfers(
conn: &mut SqliteConnection,
address: Felt,
- limit: u32,
-) -> sqlx::Result<Vec<Value>> {
- let query = format!(
+ connection: &ConnectionArguments,
+ total_count: i64,
+) -> sqlx::Result<(Vec<SqliteRow>, PageInfo)> {
+ let table_name = ERC_TRANSFER_TABLE;
+ let id_column = format!("et.{}", ID_COLUMN);
+
+ let mut query = format!(
r#"
SELECT
et.id,
@@ -83,28 +110,134 @@ SELECT
t.decimals,
c.contract_type
FROM
- erc_transfers et
+ {table_name} et
JOIN
tokens t ON et.token_id = t.id
JOIN
contracts c ON t.contract_address = c.contract_address
-WHERE
- et.from_address = ? OR et.to_address = ?
-ORDER BY
- et.executed_at DESC
-LIMIT {};
"#,
- limit
);
- let address = felt_to_sql_string(&address);
- let rows = sqlx::query(&query).bind(&address).bind(&address).fetch_all(conn).await?;
+ let mut conditions = vec!["et.from_address = ? OR et.to_address = ?".to_string()];
+
+ let mut cursor_param = &connection.after;
+ if let Some(after_cursor) = &connection.after {
+ conditions.push(handle_cursor(after_cursor, CursorDirection::After, ID_COLUMN)?);
+ }
+
+ if let Some(before_cursor) = &connection.before {
+ cursor_param = &connection.before;
+ conditions.push(handle_cursor(before_cursor, CursorDirection::Before, ID_COLUMN)?);
+ }
+
+ if !conditions.is_empty() {
+ query.push_str(&format!(" WHERE {}", conditions.join(" AND ")));
+ }
+
+ let is_cursor_based = connection.first.or(connection.last).is_some() || cursor_param.is_some();
+
+ let data_limit =
+ connection.first.or(connection.last).or(connection.limit).unwrap_or(DEFAULT_LIMIT);
+ let limit = if is_cursor_based {
+ match &cursor_param {
+ Some(_) => data_limit + 2,
+ None => data_limit + 1, // prev page does not exist
+ }
+ } else {
+ data_limit
+ };
+
+ let order_direction = match (connection.first, connection.last) {
+ (Some(_), _) => Direction::Desc,
+ (_, Some(_)) => Direction::Asc,
+ _ => Direction::Desc,
+ };
+
+ query.push_str(&format!(" ORDER BY {id_column} {} LIMIT {limit}", order_direction.as_ref()));
+
+ if let Some(offset) = connection.offset {
+ query.push_str(&format!(" OFFSET {}", offset));
+ }
+
+ let mut data = sqlx::query(&query)
+ .bind(felt_to_sql_string(&address))
+ .bind(felt_to_sql_string(&address))
+ .fetch_all(conn)
+ .await?;
+
+ let mut page_info = PageInfo {
+ has_previous_page: false,
+ has_next_page: false,
+ start_cursor: None,
+ end_cursor: None,
+ };
+
+ if data.is_empty() {
+ Ok((data, page_info))
+ } else if is_cursor_based {
+ match cursor_param {
+ Some(cursor_query) => {
+ let first_cursor = cursor::encode(
+ &data[0].try_get::<String, &str>(ID_COLUMN)?,
+ &data[0].try_get_unchecked::<String, &str>(ID_COLUMN)?,
+ );
+
+ if &first_cursor == cursor_query && data.len() != 1 {
+ data.remove(0);
+ page_info.has_previous_page = true;
+ } else {
+ data.pop();
+ }
+
+ if data.len() as u64 == limit - 1 {
+ page_info.has_next_page = true;
+ data.pop();
+ }
+ }
+ None => {
+ if data.len() as u64 == limit {
+ page_info.has_next_page = true;
+ data.pop();
+ }
+ }
+ }
+
+ if !data.is_empty() {
+ page_info.start_cursor = Some(cursor::encode(
+ &data[0].try_get::<String, &str>(ID_COLUMN)?,
+ &data[0].try_get_unchecked::<String, &str>(ID_COLUMN)?,
+ ));
+ page_info.end_cursor = Some(cursor::encode(
+ &data[data.len() - 1].try_get::<String, &str>(ID_COLUMN)?,
+ &data[data.len() - 1].try_get_unchecked::<String, &str>(ID_COLUMN)?,
+ ));
+ }
+
+ Ok((data, page_info))
+ } else {
+ let offset = connection.offset.unwrap_or(0);
+ if 1 < offset && offset < total_count as u64 {
+ page_info.has_previous_page = true;
+ }
+ if limit + offset < total_count as u64 {
+ page_info.has_next_page = true;
+ }
+
+ Ok((data, page_info))
+ }
+}
- let mut erc_balances = Vec::new();
+fn erc_transfer_connection_output(
+ data: &[SqliteRow],
+ total_count: i64,
+ page_info: PageInfo,
+) -> sqlx::Result<ValueMapping> {
+ let mut edges = Vec::new();
- for row in rows {
- let row = TransferQueryResultRaw::from_row(&row)?;
+ for row in data {
+ let row = TransferQueryResultRaw::from_row(row)?;
let transaction_hash = get_transaction_hash_from_event_id(&row.id);
+ let cursor = cursor::encode(&row.id, &row.id);
let transfer_value = match row.contract_type.to_lowercase().as_str() {
"erc20" => {
@@ -156,10 +289,17 @@ LIMIT {};
}
};
- erc_balances.push(transfer_value);
+ edges.push(Value::Object(ValueMapping::from([
+ (Name::new("node"), transfer_value),
+ (Name::new("cursor"), Value::String(cursor)),
+ ])));
}
- Ok(erc_balances)
+ Ok(ValueMapping::from([
+ (Name::new("totalCount"), Value::from(total_count)),
+ (Name::new("edges"), Value::List(edges)),
+ (Name::new("pageInfo"), PageInfoObject::value(page_info)),
+ ]))
}
// TODO: This would be required when subscriptions are needed
diff --git a/crates/torii/graphql/src/object/erc/mod.rs b/crates/torii/graphql/src/object/erc/mod.rs
index eac2c5510b..3e85722cca 100644
--- a/crates/torii/graphql/src/object/erc/mod.rs
+++ b/crates/torii/graphql/src/object/erc/mod.rs
@@ -1,3 +1,17 @@
+use super::connection::cursor;
+use crate::query::order::CursorDirection;
+
pub mod erc_balance;
pub mod erc_token;
pub mod erc_transfer;
+
+fn handle_cursor(
+ cursor: &str,
+ direction: CursorDirection,
+ id_column: &str,
+) -> sqlx::Result<String> {
+ match cursor::decode(cursor) {
+ Ok((event_id, _)) => Ok(format!("{} {} '{}'", id_column, direction.as_ref(), event_id)),
+ Err(_) => Err(sqlx::Error::Decode("Invalid cursor format".into())),
+ }
+}
From 76240e31cf8ee4230585a4e0422bdd2e1da47933 Mon Sep 17 00:00:00 2001
From: lambda-0x <0xlambda@protonmail.com>
Date: Wed, 16 Oct 2024 15:44:31 +0530
Subject: [PATCH 2/9] feat(torii): fetch and process erc721 metadata and image
commit-id:274aaa3a
---
Cargo.lock | 454 +++++++++++++++++-
Cargo.toml | 6 +
bin/torii/Cargo.toml | 6 +-
bin/torii/src/main.rs | 130 ++++-
crates/dojo-world/Cargo.toml | 55 +++
crates/sozo/ops/Cargo.toml | 8 +-
crates/torii/core/Cargo.toml | 6 +-
crates/torii/core/src/engine.rs | 12 +-
crates/torii/core/src/executor/erc.rs | 297 ++++++++++++
.../core/src/{executor.rs => executor/mod.rs} | 390 +++++++++------
.../src/processors/erc721_legacy_transfer.rs | 14 +-
.../core/src/processors/erc721_transfer.rs | 14 +-
.../core/src/processors/metadata_update.rs | 38 +-
crates/torii/core/src/sql/erc.rs | 144 ++----
crates/torii/core/src/sql/mod.rs | 6 +
crates/torii/core/src/sql/test.rs | 12 +-
crates/torii/core/src/utils.rs | 40 ++
.../torii/graphql/src/tests/metadata_test.rs | 15 +-
crates/torii/graphql/src/tests/mod.rs | 3 +-
.../graphql/src/tests/subscription_test.rs | 31 +-
.../grpc/src/server/tests/entities_test.rs | 4 +-
crates/torii/libp2p/Cargo.toml | 2 +-
crates/torii/libp2p/src/tests.rs | 6 +-
.../20241014085532_add_metadata_field.sql | 1 +
crates/torii/server/Cargo.toml | 17 +-
crates/torii/server/src/artifacts.rs | 336 +++++++++++++
crates/torii/server/src/lib.rs | 1 +
crates/torii/server/src/proxy.rs | 36 +-
28 files changed, 1738 insertions(+), 346 deletions(-)
create mode 100644 crates/dojo-world/Cargo.toml
create mode 100644 crates/torii/core/src/executor/erc.rs
rename crates/torii/core/src/{executor.rs => executor/mod.rs} (64%)
create mode 100644 crates/torii/migrations/20241014085532_add_metadata_field.sql
create mode 100644 crates/torii/server/src/artifacts.rs
diff --git a/Cargo.lock b/Cargo.lock
index 3af88cd098..e4f92c298e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -68,6 +68,12 @@ dependencies = [
"gimli",
]
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
[[package]]
name = "adler2"
version = "2.0.0"
@@ -142,6 +148,12 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "aligned-vec"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1"
+
[[package]]
name = "alloc-no-stdlib"
version = "2.0.4"
@@ -997,6 +1009,17 @@ version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
+[[package]]
+name = "arg_enum_proc_macro"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.77",
+]
+
[[package]]
name = "ark-ec"
version = "0.4.2"
@@ -1669,6 +1692,29 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
+[[package]]
+name = "av1-grain"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6678909d8c5d46a42abcf571271e15fdbc0a225e3646cf23762cd415046c78bf"
+dependencies = [
+ "anyhow",
+ "arrayvec 0.7.6",
+ "log",
+ "nom",
+ "num-rational",
+ "v_frame",
+]
+
+[[package]]
+name = "avif-serialize"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "876c75a42f6364451a033496a14c44bffe41f5f4a8236f697391f11024e596d2"
+dependencies = [
+ "arrayvec 0.7.6",
+]
+
[[package]]
name = "aws-lc-rs"
version = "1.9.0"
@@ -1805,7 +1851,7 @@ dependencies = [
"addr2line",
"cfg-if",
"libc",
- "miniz_oxide",
+ "miniz_oxide 0.8.0",
"object",
"rustc-demangle",
"windows-targets 0.52.6",
@@ -1973,6 +2019,12 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
+[[package]]
+name = "bit_field"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61"
+
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -1988,6 +2040,12 @@ dependencies = [
"serde",
]
+[[package]]
+name = "bitstream-io"
+version = "2.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b81e1519b0d82120d2fd469d5bfb2919a9361c48b02d82d04befc1cdd2002452"
+
[[package]]
name = "bitvec"
version = "1.0.1"
@@ -2195,6 +2253,12 @@ dependencies = [
"serde",
]
+[[package]]
+name = "built"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "236e6289eda5a812bc6b53c3b024039382a2895fbbeef2d748b2931546d392c4"
+
[[package]]
name = "bumpalo"
version = "3.16.0"
@@ -2258,6 +2322,12 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+[[package]]
+name = "byteorder-lite"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
+
[[package]]
name = "bytes"
version = "1.7.2"
@@ -3394,7 +3464,7 @@ checksum = "9a95746c5221a74d7b913a415fdbb9e7c90e1b4d818dbbff59bddc034cfce2ec"
dependencies = [
"bytes",
"flex-error",
- "num-derive",
+ "num-derive 0.3.3",
"num-traits 0.2.19",
"prost 0.12.6",
"prost-types 0.12.6",
@@ -3448,6 +3518,16 @@ dependencies = [
"nom",
]
+[[package]]
+name = "cfg-expr"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02"
+dependencies = [
+ "smallvec",
+ "target-lexicon",
+]
+
[[package]]
name = "cfg-if"
version = "1.0.0"
@@ -3655,6 +3735,12 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15"
+[[package]]
+name = "color_quant"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
+
[[package]]
name = "colorchoice"
version = "1.0.2"
@@ -4311,6 +4397,12 @@ dependencies = [
"syn 1.0.109",
]
+[[package]]
+name = "data-url"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a"
+
[[package]]
name = "debugid"
version = "0.8.0"
@@ -5163,6 +5255,22 @@ dependencies = [
"pin-project-lite",
]
+[[package]]
+name = "exr"
+version = "1.72.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "887d93f60543e9a9362ef8a21beedd0a833c5d9610e18c67abe15a5963dcb1a4"
+dependencies = [
+ "bit_field",
+ "flume",
+ "half 2.4.1",
+ "lebe",
+ "miniz_oxide 0.7.4",
+ "rayon-core",
+ "smallvec",
+ "zune-inflate",
+]
+
[[package]]
name = "eyre"
version = "0.6.12"
@@ -5214,6 +5322,15 @@ dependencies = [
"bytes",
]
+[[package]]
+name = "fdeflate"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8090f921a24b04994d9929e204f50b498a33ea6ba559ffaa05e04f7ee7fb5ab"
+dependencies = [
+ "simd-adler32",
+]
+
[[package]]
name = "ff"
version = "0.13.0"
@@ -5279,7 +5396,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253"
dependencies = [
"crc32fast",
- "miniz_oxide",
+ "miniz_oxide 0.8.0",
]
[[package]]
@@ -5605,6 +5722,16 @@ dependencies = [
"polyval",
]
+[[package]]
+name = "gif"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2"
+dependencies = [
+ "color_quant",
+ "weezl",
+]
+
[[package]]
name = "gimli"
version = "0.31.0"
@@ -7300,6 +7427,39 @@ dependencies = [
"winapi-util",
]
+[[package]]
+name = "image"
+version = "0.25.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10"
+dependencies = [
+ "bytemuck",
+ "byteorder-lite",
+ "color_quant",
+ "exr",
+ "gif",
+ "image-webp",
+ "num-traits 0.2.19",
+ "png",
+ "qoi",
+ "ravif",
+ "rayon",
+ "rgb",
+ "tiff",
+ "zune-core",
+ "zune-jpeg",
+]
+
+[[package]]
+name = "image-webp"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904"
+dependencies = [
+ "byteorder-lite",
+ "quick-error 2.0.1",
+]
+
[[package]]
name = "imara-diff"
version = "0.1.7"
@@ -7310,6 +7470,12 @@ dependencies = [
"hashbrown 0.14.5",
]
+[[package]]
+name = "imgref"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44feda355f4159a7c757171a77de25daf6411e217b4cabd03bd6650690468126"
+
[[package]]
name = "impl-codec"
version = "0.6.0"
@@ -7502,6 +7668,17 @@ dependencies = [
"webrtc-util 0.8.1",
]
+[[package]]
+name = "interpolate_name"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.77",
+]
+
[[package]]
name = "io-close"
version = "0.3.7"
@@ -7752,6 +7929,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "jpeg-decoder"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0"
+
[[package]]
name = "js-sys"
version = "0.3.70"
@@ -8655,6 +8838,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+[[package]]
+name = "lebe"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
+
[[package]]
name = "leopard-codec"
version = "0.1.0"
@@ -8672,6 +8861,17 @@ version = "0.2.158"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
+[[package]]
+name = "libfuzzer-sys"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a96cfd5557eb82f2b83fed4955246c988d331975a002961b07c81584d107e7f7"
+dependencies = [
+ "arbitrary",
+ "cc",
+ "once_cell",
+]
+
[[package]]
name = "libloading"
version = "0.8.5"
@@ -9280,6 +9480,15 @@ dependencies = [
"value-bag",
]
+[[package]]
+name = "loop9"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062"
+dependencies = [
+ "imgref",
+]
+
[[package]]
name = "lru"
version = "0.12.4"
@@ -9359,6 +9568,15 @@ dependencies = [
"rawpointer",
]
+[[package]]
+name = "maybe-rayon"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519"
+dependencies = [
+ "cfg-if",
+]
+
[[package]]
name = "md-5"
version = "0.10.6"
@@ -9512,6 +9730,15 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+[[package]]
+name = "miniz_oxide"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
+dependencies = [
+ "adler",
+]
+
[[package]]
name = "miniz_oxide"
version = "0.8.0"
@@ -9519,6 +9746,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
dependencies = [
"adler2",
+ "simd-adler32",
]
[[package]]
@@ -9865,6 +10093,12 @@ dependencies = [
"minimal-lexical",
]
+[[package]]
+name = "noop_proc_macro"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8"
+
[[package]]
name = "notify"
version = "7.0.0"
@@ -9968,6 +10202,17 @@ dependencies = [
"syn 1.0.109",
]
+[[package]]
+name = "num-derive"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.77",
+]
+
[[package]]
name = "num-format"
version = "0.4.4"
@@ -10666,6 +10911,19 @@ dependencies = [
"plotters-backend",
]
+[[package]]
+name = "png"
+version = "0.17.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52f9d46a34a05a6a57566bc2bfae066ef07585a6e3fa30fbbdff5936380623f0"
+dependencies = [
+ "bitflags 1.3.2",
+ "crc32fast",
+ "fdeflate",
+ "flate2",
+ "miniz_oxide 0.8.0",
+]
+
[[package]]
name = "polling"
version = "2.8.0"
@@ -10966,6 +11224,25 @@ dependencies = [
"human_format",
]
+[[package]]
+name = "profiling"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "afbdc74edc00b6f6a218ca6a5364d6226a259d4b8ea1af4a0ea063f27e179f4d"
+dependencies = [
+ "profiling-procmacros",
+]
+
+[[package]]
+name = "profiling-procmacros"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30"
+dependencies = [
+ "quote",
+ "syn 2.0.77",
+]
+
[[package]]
name = "prometheus-client"
version = "0.22.3"
@@ -11184,6 +11461,15 @@ dependencies = [
"psl-types",
]
+[[package]]
+name = "qoi"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001"
+dependencies = [
+ "bytemuck",
+]
+
[[package]]
name = "quanta"
version = "0.12.3"
@@ -11205,6 +11491,12 @@ version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
+[[package]]
+name = "quick-error"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
+
[[package]]
name = "quick-protobuf"
version = "0.8.1"
@@ -11386,6 +11678,55 @@ dependencies = [
"rand_core 0.6.4",
]
+[[package]]
+name = "rav1e"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9"
+dependencies = [
+ "arbitrary",
+ "arg_enum_proc_macro",
+ "arrayvec 0.7.6",
+ "av1-grain",
+ "bitstream-io",
+ "built",
+ "cfg-if",
+ "interpolate_name",
+ "itertools 0.12.1",
+ "libc",
+ "libfuzzer-sys",
+ "log",
+ "maybe-rayon",
+ "new_debug_unreachable",
+ "noop_proc_macro",
+ "num-derive 0.4.2",
+ "num-traits 0.2.19",
+ "once_cell",
+ "paste",
+ "profiling",
+ "rand 0.8.5",
+ "rand_chacha",
+ "simd_helpers",
+ "system-deps",
+ "thiserror",
+ "v_frame",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "ravif"
+version = "0.11.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f0bfd976333248de2078d350bfdf182ff96e168a24d23d2436cef320dd4bdd"
+dependencies = [
+ "avif-serialize",
+ "imgref",
+ "loop9",
+ "quick-error 2.0.1",
+ "rav1e",
+ "rgb",
+]
+
[[package]]
name = "raw-cpuid"
version = "11.1.0"
@@ -11673,7 +12014,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00"
dependencies = [
"hostname",
- "quick-error",
+ "quick-error 1.2.3",
]
[[package]]
@@ -12302,7 +12643,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f"
dependencies = [
"fnv",
- "quick-error",
+ "quick-error 1.2.3",
"tempfile",
"wait-timeout",
]
@@ -13137,6 +13478,21 @@ dependencies = [
"rand_core 0.6.4",
]
+[[package]]
+name = "simd-adler32"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
+
+[[package]]
+name = "simd_helpers"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6"
+dependencies = [
+ "quote",
+]
+
[[package]]
name = "simdutf8"
version = "0.1.5"
@@ -14379,6 +14735,19 @@ dependencies = [
"libc",
]
+[[package]]
+name = "system-deps"
+version = "6.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349"
+dependencies = [
+ "cfg-expr",
+ "heck 0.5.0",
+ "pkg-config",
+ "toml 0.8.19",
+ "version-compare",
+]
+
[[package]]
name = "tabled"
version = "0.16.0"
@@ -14421,6 +14790,12 @@ dependencies = [
"xattr",
]
+[[package]]
+name = "target-lexicon"
+version = "0.12.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+
[[package]]
name = "tempdir"
version = "0.3.7"
@@ -14530,6 +14905,17 @@ dependencies = [
"num_cpus",
]
+[[package]]
+name = "tiff"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e"
+dependencies = [
+ "flate2",
+ "jpeg-decoder",
+ "weezl",
+]
+
[[package]]
name = "time"
version = "0.3.36"
@@ -15027,6 +15413,7 @@ dependencies = [
"cainome 0.4.6",
"chrono",
"crypto-bigint",
+ "data-url",
"dojo-test-utils",
"dojo-types 1.0.0",
"dojo-utils",
@@ -15034,6 +15421,7 @@ dependencies = [
"futures-channel",
"futures-util",
"hashlink",
+ "ipfs-api-backend-hyper",
"katana-runner",
"num-traits 0.2.19",
"once_cell",
@@ -15179,20 +15567,29 @@ dependencies = [
name = "torii-server"
version = "1.0.0"
dependencies = [
+ "anyhow",
"base64 0.21.7",
+ "camino",
+ "data-url",
"http 0.2.12",
"http-body 0.4.6",
"hyper 0.14.30",
"hyper-reverse-proxy",
+ "image",
"indexmap 2.5.0",
"lazy_static",
+ "mime_guess",
+ "reqwest 0.11.27",
"serde",
"serde_json",
+ "sqlx",
"tokio",
"tokio-util",
+ "torii-core",
"tower 0.4.13",
"tower-http 0.4.4",
"tracing",
+ "warp",
]
[[package]]
@@ -15776,6 +16173,17 @@ dependencies = [
"getrandom",
]
+[[package]]
+name = "v_frame"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b"
+dependencies = [
+ "aligned-vec",
+ "num-traits 0.2.19",
+ "wasm-bindgen",
+]
+
[[package]]
name = "valuable"
version = "0.1.0"
@@ -15849,6 +16257,12 @@ dependencies = [
"tracing-subscriber",
]
+[[package]]
+name = "version-compare"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b"
+
[[package]]
name = "version_check"
version = "0.9.5"
@@ -16387,6 +16801,12 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "weezl"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082"
+
[[package]]
name = "which"
version = "4.4.2"
@@ -17082,3 +17502,27 @@ dependencies = [
"cc",
"pkg-config",
]
+
+[[package]]
+name = "zune-core"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a"
+
+[[package]]
+name = "zune-inflate"
+version = "0.2.54"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02"
+dependencies = [
+ "simd-adler32",
+]
+
+[[package]]
+name = "zune-jpeg"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16099418600b4d8f028622f73ff6e3deaabdff330fb9a2a131dea781ee8b0768"
+dependencies = [
+ "zune-core",
+]
diff --git a/Cargo.toml b/Cargo.toml
index a5b69bf2e2..fa4a365206 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -169,14 +169,17 @@ colored_json = "3.2.0"
console = "0.15.7"
convert_case = "0.6.0"
crypto-bigint = { version = "0.5.3", features = [ "serde" ] }
+data-url = "0.3"
derive_more = "0.99.17"
flate2 = "1.0.24"
+fluent-uri = "0.3"
futures = "0.3.30"
futures-util = "0.3.30"
hashlink = "0.9.1"
hex = "0.4.3"
hex-literal = "0.4.1"
http = "0.2.9"
+image = "0.25.2"
indexmap = "2.2.5"
indoc = "1.0.7"
itertools = "0.12.1"
@@ -224,6 +227,9 @@ tracing-log = "0.1.3"
tracing-subscriber = { version = "0.3.16", features = [ "env-filter", "json" ] }
url = { version = "2.4.0", features = [ "serde" ] }
walkdir = "2.5.0"
+# TODO: see if we still need the git version
+ipfs-api-backend-hyper = { git = "https://github.com/ferristseng/rust-ipfs-api", rev = "af2c17f7b19ef5b9898f458d97a90055c3605633", features = [ "with-hyper-rustls", "with-send-sync" ] }
+mime_guess = "2.0"
# server
hyper = "0.14.27"
diff --git a/bin/torii/Cargo.toml b/bin/torii/Cargo.toml
index b9bf3cf1cf..2e652a89ca 100644
--- a/bin/torii/Cargo.toml
+++ b/bin/torii/Cargo.toml
@@ -33,6 +33,7 @@ starknet.workspace = true
tokio-stream = "0.1.11"
tokio-util = "0.7.7"
tokio.workspace = true
+toml.workspace = true
torii-cli.workspace = true
torii-core.workspace = true
torii-graphql.workspace = true
@@ -40,15 +41,14 @@ torii-grpc = { workspace = true, features = [ "server" ] }
torii-relay.workspace = true
torii-server.workspace = true
tower.workspace = true
-toml.workspace = true
+clap_config = "0.1.1"
+tempfile.workspace = true
tower-http.workspace = true
tracing-subscriber.workspace = true
tracing.workspace = true
url.workspace = true
webbrowser = "0.8"
-tempfile.workspace = true
-clap_config = "0.1.1"
[dev-dependencies]
assert_matches.workspace = true
diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs
index 57415c48e6..1d5b2f1322 100644
--- a/bin/torii/src/main.rs
+++ b/bin/torii/src/main.rs
@@ -16,7 +16,10 @@ use std::str::FromStr;
use std::sync::Arc;
use std::time::Duration;
+use anyhow::Context;
+use camino::Utf8PathBuf;
use clap::Parser;
+use clap::{ArgAction, Parser};
use dojo_metrics::exporters::prometheus::PrometheusRecorder;
use dojo_world::contracts::world::WorldContractReader;
use sqlx::sqlite::{
@@ -25,7 +28,7 @@ use sqlx::sqlite::{
use sqlx::SqlitePool;
use starknet::providers::jsonrpc::HttpTransport;
use starknet::providers::JsonRpcClient;
-use tempfile::NamedTempFile;
+use tempfile::{NamedTempFile, TempDir};
use tokio::sync::broadcast;
use tokio::sync::broadcast::Sender;
use tokio_stream::StreamExt;
@@ -45,6 +48,111 @@ use url::{form_urlencoded, Url};
pub(crate) const LOG_TARGET: &str = "torii::cli";
+/// Dojo World Indexer
+#[derive(Parser, Debug)]
+#[command(name = "torii", author, version, about, long_about = None)]
+struct Args {
+ /// The world to index
+ #[arg(short, long = "world", env = "DOJO_WORLD_ADDRESS")]
+ world_address: Option<Felt>,
+
+ /// The sequencer rpc endpoint to index.
+ #[arg(long, value_name = "URL", default_value = ":5050", value_parser = parse_url)]
+ rpc: Url,
+
+ /// Database filepath (ex: indexer.db). If specified file doesn't exist, it will be
+ /// created. Defaults to in-memory database
+ #[arg(short, long, default_value = "")]
+ database: String,
+
+ /// Address to serve api endpoints at.
+ #[arg(long, value_name = "SOCKET", default_value = "0.0.0.0:8080", value_parser = parse_socket_address)]
+ addr: SocketAddr,
+
+ /// Port to serve Libp2p TCP & UDP Quic transports
+ #[arg(long, value_name = "PORT", default_value = "9090")]
+ relay_port: u16,
+
+ /// Port to serve Libp2p WebRTC transport
+ #[arg(long, value_name = "PORT", default_value = "9091")]
+ relay_webrtc_port: u16,
+
+ /// Port to serve Libp2p WebRTC transport
+ #[arg(long, value_name = "PORT", default_value = "9092")]
+ relay_websocket_port: u16,
+
+ /// Path to a local identity key file. If not specified, a new identity will be generated
+ #[arg(long, value_name = "PATH")]
+ relay_local_key_path: Option<String>,
+
+ /// Path to a local certificate file. If not specified, a new certificate will be generated
+ /// for WebRTC connections
+ #[arg(long, value_name = "PATH")]
+ relay_cert_path: Option<String>,
+
+ /// Specify allowed origins for api endpoints (comma-separated list of allowed origins, or "*"
+ /// for all)
+ #[arg(long)]
+ #[arg(value_delimiter = ',')]
+ allowed_origins: Option<Vec<String>>,
+
+ /// The external url of the server, used for configuring the GraphQL Playground in a hosted
+ /// environment
+ #[arg(long, value_parser = parse_url)]
+ external_url: Option<Url>,
+
+ /// Enable Prometheus metrics.
+ ///
+ /// The metrics will be served at the given interface and port.
+ #[arg(long, value_name = "SOCKET", value_parser = parse_socket_address, help_heading = "Metrics")]
+ metrics: Option<SocketAddr>,
+
+ /// Open World Explorer on the browser.
+ #[arg(long)]
+ explorer: bool,
+
+ /// Chunk size of the events page when indexing using events
+ #[arg(long, default_value = "1024")]
+ events_chunk_size: u64,
+
+ /// Number of blocks to process before commiting to DB
+ #[arg(long, default_value = "10240")]
+ blocks_chunk_size: u64,
+
+ /// Enable indexing pending blocks
+ #[arg(long, action = ArgAction::Set, default_value_t = true)]
+ index_pending: bool,
+
+ /// Polling interval in ms
+ #[arg(long, default_value = "500")]
+ polling_interval: u64,
+
+ /// Max concurrent tasks
+ #[arg(long, default_value = "100")]
+ max_concurrent_tasks: usize,
+
+ /// Whether or not to index world transactions
+ #[arg(long, action = ArgAction::Set, default_value_t = false)]
+ index_transactions: bool,
+
+ /// Whether or not to index raw events
+ #[arg(long, action = ArgAction::Set, default_value_t = true)]
+ index_raw_events: bool,
+
+ /// ERC contract addresses to index
+ #[arg(long, value_parser = parse_erc_contracts)]
+ #[arg(conflicts_with = "config")]
+ contracts: Option<std::vec::Vec<Contract>>,
+
+ /// Configuration file
+ #[arg(long)]
+ config: Option<PathBuf>,
+
+ /// Path to a directory to store ERC artifacts
+ #[arg(long)]
+ artifacts_path: Option<Utf8PathBuf>,
+}
+
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let mut args = ToriiArgs::parse().with_config_file()?;
@@ -109,7 +217,21 @@ async fn main() -> anyhow::Result<()> {
// Get world address
let world = WorldContractReader::new(world_address, provider.clone());
- let (mut executor, sender) = Executor::new(pool.clone(), shutdown_tx.clone()).await?;
+ // let (mut executor, sender) = Executor::new(pool.clone(), shutdown_tx.clone()).await?;
+ let contracts = args
+ .indexing
+ .contracts
+ .iter()
+ .map(|contract| (contract.address, contract.r#type))
+ .collect();
+
+ let (mut executor, sender) = Executor::new(
+ pool.clone(),
+ shutdown_tx.clone(),
+ provider.clone(),
+ args.max_concurrent_tasks,
+ )
+ .await?;
tokio::spawn(async move {
executor.run().await.unwrap();
});
@@ -184,6 +306,7 @@ async fn main() -> anyhow::Result<()> {
args.server.http_cors_origins.filter(|cors_origins| !cors_origins.is_empty()),
Some(grpc_addr),
None,
+ Some(artifacts_addr),
));
let graphql_server = spawn_rebuilding_graphql_server(
@@ -201,6 +324,7 @@ async fn main() -> anyhow::Result<()> {
info!(target: LOG_TARGET, endpoint = %addr, "Starting torii endpoint.");
info!(target: LOG_TARGET, endpoint = %gql_endpoint, "Serving Graphql playground.");
info!(target: LOG_TARGET, url = %explorer_url, "Serving World Explorer.");
+ info!(target: LOG_TARGET, path = %artifacts_path, "Serving ERC artifacts at path");
if args.explorer {
if let Err(e) = webbrowser::open(&explorer_url) {
@@ -222,6 +346,7 @@ async fn main() -> anyhow::Result<()> {
let graphql_server_handle = tokio::spawn(graphql_server);
let grpc_server_handle = tokio::spawn(grpc_server);
let libp2p_relay_server_handle = tokio::spawn(async move { libp2p_relay_server.run().await });
+ let artifacts_server_handle = tokio::spawn(artifacts_server);
tokio::select! {
res = engine_handle => res??,
@@ -229,6 +354,7 @@ async fn main() -> anyhow::Result<()> {
res = graphql_server_handle => res?,
res = grpc_server_handle => res??,
res = libp2p_relay_server_handle => res?,
+ res = artifacts_server_handle => res?,
_ = dojo_utils::signal::wait_signals() => {},
};
diff --git a/crates/dojo-world/Cargo.toml b/crates/dojo-world/Cargo.toml
new file mode 100644
index 0000000000..39301853ca
--- /dev/null
+++ b/crates/dojo-world/Cargo.toml
@@ -0,0 +1,55 @@
+[package]
+description = "Dojo world specification. For example, crates and flags used for compilation."
+edition.workspace = true
+license-file.workspace = true
+name = "dojo-world"
+repository.workspace = true
+version.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+async-trait.workspace = true
+cairo-lang-filesystem.workspace = true
+cairo-lang-project.workspace = true
+cairo-lang-starknet-classes.workspace = true
+cairo-lang-starknet.workspace = true
+camino.workspace = true
+convert_case.workspace = true
+dojo-utils = { workspace = true, optional = true }
+num-traits = { workspace = true, optional = true }
+regex.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+serde_with.workspace = true
+smol_str.workspace = true
+starknet-crypto.workspace = true
+starknet.workspace = true
+thiserror.workspace = true
+topological-sort.workspace = true
+tracing.workspace = true
+
+cainome.workspace = true
+dojo-types = { path = "../dojo-types", optional = true }
+http = { workspace = true, optional = true }
+ipfs-api-backend-hyper = { workspace = true, optional = true }
+scarb = { workspace = true, optional = true }
+tokio = { version = "1.32.0", features = [ "time" ], default-features = false, optional = true }
+toml.workspace = true
+url = { workspace = true, optional = true }
+walkdir.workspace = true
+
+[dev-dependencies]
+assert_fs.workspace = true
+assert_matches.workspace = true
+dojo-lang.workspace = true
+dojo-test-utils = { path = "../dojo-test-utils" }
+katana-runner.workspace = true
+similar-asserts.workspace = true
+tempfile.workspace = true
+tokio.workspace = true
+
+[features]
+contracts = [ "dep:dojo-types", "dep:http", "dep:num-traits" ]
+manifest = [ "contracts", "dep:dojo-types", "dep:scarb", "dep:url" ]
+metadata = [ "dep:ipfs-api-backend-hyper", "dep:scarb", "dep:url" ]
+migration = [ "dep:dojo-utils", "dep:scarb", "dep:tokio", "manifest" ]
diff --git a/crates/sozo/ops/Cargo.toml b/crates/sozo/ops/Cargo.toml
index 170612e2ec..54a6982f8f 100644
--- a/crates/sozo/ops/Cargo.toml
+++ b/crates/sozo/ops/Cargo.toml
@@ -11,8 +11,8 @@ async-trait.workspace = true
cainome.workspace = true
colored.workspace = true
colored_json.workspace = true
-dojo-utils.workspace = true
dojo-types.workspace = true
+dojo-utils.workspace = true
dojo-world.workspace = true
futures.workspace = true
num-traits.workspace = true
@@ -21,8 +21,8 @@ serde_json.workspace = true
serde_with.workspace = true
sozo-walnut = { workspace = true, optional = true }
spinoff.workspace = true
-starknet.workspace = true
starknet-crypto.workspace = true
+starknet.workspace = true
thiserror.workspace = true
toml.workspace = true
tracing.workspace = true
@@ -33,11 +33,11 @@ katana-runner = { workspace = true, optional = true }
[dev-dependencies]
assert_fs.workspace = true
dojo-test-utils = { workspace = true, features = [ "build-examples" ] }
-ipfs-api-backend-hyper = { git = "https://github.com/ferristseng/rust-ipfs-api", rev = "af2c17f7b19ef5b9898f458d97a90055c3605633", features = [ "with-hyper-rustls" ] }
+ipfs-api-backend-hyper.workspace = true
katana-runner.workspace = true
-tokio.workspace = true
scarb.workspace = true
sozo-scarbext.workspace = true
+tokio.workspace = true
[features]
test-utils = [ "dep:dojo-test-utils", "dep:katana-runner" ]
diff --git a/crates/torii/core/Cargo.toml b/crates/torii/core/Cargo.toml
index aac4d9010f..09b25d8cca 100644
--- a/crates/torii/core/Cargo.toml
+++ b/crates/torii/core/Cargo.toml
@@ -16,6 +16,7 @@ bitflags = "2.6.0"
cainome.workspace = true
chrono.workspace = true
crypto-bigint.workspace = true
+data-url.workspace = true
dojo-types.workspace = true
dojo-world.workspace = true
futures-channel = "0.3.0"
@@ -31,8 +32,9 @@ sqlx.workspace = true
starknet-crypto.workspace = true
starknet.workspace = true
thiserror.workspace = true
-tokio = { version = "1.32.0", features = [ "sync", "macros" ], default-features = true }
+tokio = { version = "1.32.0", features = [ "macros", "sync" ], default-features = true }
# tokio-stream = "0.1.11"
+ipfs-api-backend-hyper.workspace = true
tokio-util.workspace = true
tracing.workspace = true
@@ -41,5 +43,5 @@ dojo-test-utils.workspace = true
dojo-utils.workspace = true
katana-runner.workspace = true
scarb.workspace = true
-tempfile.workspace = true
sozo-scarbext.workspace = true
+tempfile.workspace = true
diff --git a/crates/torii/core/src/engine.rs b/crates/torii/core/src/engine.rs
index ccac372feb..edbbea147f 100644
--- a/crates/torii/core/src/engine.rs
+++ b/crates/torii/core/src/engine.rs
@@ -277,8 +277,9 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
match self.process(fetch_result).await {
Ok(_) => {
- self.db.execute().await?;
+ self.db.flush().await?;
self.db.apply_cache_diff().await?;
+ self.db.execute().await?;
},
Err(e) => {
error!(target: LOG_TARGET, error = %e, "Processing fetched data.");
@@ -646,8 +647,7 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
unique_contracts.insert(event.from_address);
- Self::process_event(
- self,
+ self.process_event(
block_number,
block_timestamp,
&event_id,
@@ -707,8 +707,7 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
let event_id =
format!("{:#064x}:{:#x}:{:#04x}", block_number, *transaction_hash, event_idx);
- Self::process_event(
- self,
+ self.process_event(
block_number,
block_timestamp,
&event_id,
@@ -720,8 +719,7 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
}
if self.config.flags.contains(IndexingFlags::TRANSACTIONS) {
- Self::process_transaction(
- self,
+ self.process_transaction(
block_number,
block_timestamp,
*transaction_hash,
diff --git a/crates/torii/core/src/executor/erc.rs b/crates/torii/core/src/executor/erc.rs
new file mode 100644
index 0000000000..eb4a3c19bc
--- /dev/null
+++ b/crates/torii/core/src/executor/erc.rs
@@ -0,0 +1,297 @@
+use std::str::FromStr;
+use std::sync::Arc;
+
+use anyhow::{Context, Result};
+use cainome::cairo_serde::{ByteArray, CairoSerde};
+use data_url::mime::Mime;
+use data_url::DataUrl;
+use reqwest::Client;
+use starknet::core::types::{BlockId, BlockTag, FunctionCall, U256};
+use starknet::core::utils::{get_selector_from_name, parse_cairo_short_string};
+use starknet::providers::Provider;
+use starknet_crypto::Felt;
+use tracing::{debug, trace};
+
+use super::{ApplyBalanceDiffQuery, Executor};
+use crate::constants::TOKEN_BALANCE_TABLE;
+use crate::sql::utils::{felt_to_sql_string, sql_string_to_u256, u256_to_sql_string, I256};
+use crate::sql::FELT_DELIMITER;
+use crate::types::ContractType;
+use crate::utils::{fetch_content_from_ipfs, MAX_RETRY};
+
+#[derive(Debug, Clone)]
+pub struct RegisterErc721TokenQuery {
+ pub token_id: String,
+ pub contract_address: Felt,
+ pub actual_token_id: U256,
+}
+
+#[derive(Debug, Clone)]
+pub struct RegisterErc721TokenMetadata {
+ pub query: RegisterErc721TokenQuery,
+ pub name: String,
+ pub symbol: String,
+ pub metadata: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct RegisterErc20TokenQuery {
+ pub token_id: String,
+ pub contract_address: Felt,
+ pub name: String,
+ pub symbol: String,
+ pub decimals: u8,
+}
+
+impl<'c, P: Provider + Sync + Send + 'static> Executor<'c, P> {
+ pub async fn apply_balance_diff(
+ &mut self,
+ apply_balance_diff: ApplyBalanceDiffQuery,
+ ) -> Result<()> {
+ let erc_cache = apply_balance_diff.erc_cache;
+ for ((contract_type, id_str), balance) in erc_cache.iter() {
+ let id = id_str.split(FELT_DELIMITER).collect::<Vec<&str>>();
+ match contract_type {
+ ContractType::WORLD => unreachable!(),
+ ContractType::ERC721 => {
+ // account_address/contract_address:id => ERC721
+ assert!(id.len() == 2);
+ let account_address = id[0];
+ let token_id = id[1];
+ let mid = token_id.split(":").collect::<Vec<&str>>();
+ let contract_address = mid[0];
+
+ self.apply_balance_diff_helper(
+ id_str,
+ account_address,
+ contract_address,
+ token_id,
+ balance,
+ )
+ .await
+ .with_context(|| "Failed to apply balance diff in apply_cache_diff")?;
+ }
+ ContractType::ERC20 => {
+ // account_address/contract_address/ => ERC20
+ assert!(id.len() == 3);
+ let account_address = id[0];
+ let contract_address = id[1];
+ let token_id = id[1];
+
+ self.apply_balance_diff_helper(
+ id_str,
+ account_address,
+ contract_address,
+ token_id,
+ balance,
+ )
+ .await
+ .with_context(|| "Failed to apply balance diff in apply_cache_diff")?;
+ }
+ }
+ }
+
+ Ok(())
+ }
+
+ pub async fn apply_balance_diff_helper(
+ &mut self,
+ id: &str,
+ account_address: &str,
+ contract_address: &str,
+ token_id: &str,
+ balance_diff: &I256,
+ ) -> Result<()> {
+ let tx = &mut self.transaction;
+ let balance: Option<(String,)> =
+ sqlx::query_as(&format!("SELECT balance FROM {TOKEN_BALANCE_TABLE} WHERE id = ?"))
+ .bind(id)
+ .fetch_optional(&mut **tx)
+ .await?;
+
+ let mut balance = if let Some(balance) = balance {
+ sql_string_to_u256(&balance.0)
+ } else {
+ U256::from(0u8)
+ };
+
+ if balance_diff.is_negative {
+ if balance < balance_diff.value {
+ dbg!(&balance_diff, balance, id);
+ }
+ balance -= balance_diff.value;
+ } else {
+ balance += balance_diff.value;
+ }
+
+ // write the new balance to the database
+ sqlx::query(&format!(
+ "INSERT OR REPLACE INTO {TOKEN_BALANCE_TABLE} (id, contract_address, account_address, \
+ token_id, balance) VALUES (?, ?, ?, ?, ?)",
+ ))
+ .bind(id)
+ .bind(contract_address)
+ .bind(account_address)
+ .bind(token_id)
+ .bind(u256_to_sql_string(&balance))
+ .execute(&mut **tx)
+ .await?;
+
+ Ok(())
+ }
+
+ pub async fn process_register_erc721_token_query(
+ register_erc721_token: RegisterErc721TokenQuery,
+ provider: Arc<P>,
+ name: String,
+ symbol: String,
+ ) -> Result<RegisterErc721TokenMetadata> {
+ let token_uri = if let Ok(token_uri) = provider
+ .call(
+ FunctionCall {
+ contract_address: register_erc721_token.contract_address,
+ entry_point_selector: get_selector_from_name("token_uri").unwrap(),
+ calldata: vec![
+ register_erc721_token.actual_token_id.low().into(),
+ register_erc721_token.actual_token_id.high().into(),
+ ],
+ },
+ BlockId::Tag(BlockTag::Pending),
+ )
+ .await
+ {
+ token_uri
+ } else if let Ok(token_uri) = provider
+ .call(
+ FunctionCall {
+ contract_address: register_erc721_token.contract_address,
+ entry_point_selector: get_selector_from_name("tokenURI").unwrap(),
+ calldata: vec![
+ register_erc721_token.actual_token_id.low().into(),
+ register_erc721_token.actual_token_id.high().into(),
+ ],
+ },
+ BlockId::Tag(BlockTag::Pending),
+ )
+ .await
+ {
+ token_uri
+ } else {
+ return Err(anyhow::anyhow!("Failed to fetch token_uri"));
+ };
+
+ let token_uri = if let Ok(byte_array) = ByteArray::cairo_deserialize(&token_uri, 0) {
+ byte_array.to_string().expect("Return value not String")
+ } else if let Ok(felt_array) = Vec::<Felt>::cairo_deserialize(&token_uri, 0) {
+ felt_array
+ .iter()
+ .map(parse_cairo_short_string)
+ .collect::<Result<Vec<String>, _>>()
+ .map(|strings| strings.join(""))
+ .map_err(|_| anyhow::anyhow!("Failed parsing Array<Felt> to String"))?
+ } else {
+ return Err(anyhow::anyhow!("token_uri is neither ByteArray nor Array<Felt>"));
+ };
+
+ let metadata = Self::fetch_metadata(&token_uri).await.with_context(|| {
+ format!(
+ "Failed to fetch metadata for token_id: {}",
+ register_erc721_token.actual_token_id
+ )
+ })?;
+ let metadata = serde_json::to_string(&metadata).context("Failed to serialize metadata")?;
+ Ok(RegisterErc721TokenMetadata { query: register_erc721_token, metadata, name, symbol })
+ }
+
+ // given a uri which can be either http/https url or data uri, fetch the metadata erc721
+ // metadata json schema
+ pub async fn fetch_metadata(token_uri: &str) -> Result<serde_json::Value> {
+ // Parse the token_uri
+
+ match token_uri {
+ uri if uri.starts_with("http") || uri.starts_with("https") => {
+ // Fetch metadata from HTTP/HTTPS URL
+ debug!(token_uri = %token_uri, "Fetching metadata from http/https URL");
+ let client = Client::new();
+ let response = client
+ .get(token_uri)
+ .send()
+ .await
+ .context("Failed to fetch metadata from URL")?;
+
+ let bytes = response.bytes().await.context("Failed to read response bytes")?;
+ let json: serde_json::Value = serde_json::from_slice(&bytes)
+ .context(format!("Failed to parse metadata JSON from response: {:?}", bytes))?;
+
+ Ok(json)
+ }
+ uri if uri.starts_with("ipfs") => {
+ let cid = uri.strip_prefix("ipfs://").unwrap();
+ debug!(cid = %cid, "Fetching metadata from IPFS");
+ let response = fetch_content_from_ipfs(cid, MAX_RETRY)
+ .await
+ .context("Failed to fetch metadata from IPFS")?;
+
+ let json: serde_json::Value =
+ serde_json::from_slice(&response).context(format!(
+ "Failed to parse metadata JSON from IPFS: {:?}, data: {:?}",
+ cid, &response
+ ))?;
+
+ Ok(json)
+ }
+ uri if uri.starts_with("data") => {
+ // Parse and decode data URI
+ debug!("Parsing metadata from data URI");
+ trace!(data_uri = %token_uri);
+
+ // HACK: https://github.com/servo/rust-url/issues/908
+ let uri = token_uri.replace("#", "%23");
+
+ let data_url = DataUrl::process(&uri).context("Failed to parse data URI")?;
+
+ // Ensure the MIME type is JSON
+ if data_url.mime_type() != &Mime::from_str("application/json").unwrap() {
+ return Err(anyhow::anyhow!("Data URI is not of JSON type"));
+ }
+
+ let decoded = data_url.decode_to_vec().context("Failed to decode data URI")?;
+ // HACK: Loot Survior NFT metadata contains control characters which makes the json
+ // DATA invalid so filter them out
+ let decoded_str = String::from_utf8_lossy(&decoded.0)
+ .chars()
+ .filter(|c| !c.is_ascii_control())
+ .collect::<String>();
+
+ let json: serde_json::Value = serde_json::from_str(&decoded_str)
+ .context(format!("Failed to parse metadata JSON from data URI: {}", &uri))?;
+
+ Ok(json)
+ }
+ uri => Err(anyhow::anyhow!("Unsupported URI scheme found in token URI: {}", uri)),
+ }
+ }
+
+ pub async fn handle_erc721_token_metadata(
+ &mut self,
+ result: RegisterErc721TokenMetadata,
+ ) -> Result<()> {
+ let query = sqlx::query(
+ "INSERT INTO tokens (id, contract_address, name, symbol, decimals, metadata) VALUES \
+ (?, ?, ?, ?, ?, ?)",
+ )
+ .bind(&result.query.token_id)
+ .bind(felt_to_sql_string(&result.query.contract_address))
+ .bind(&result.name)
+ .bind(&result.symbol)
+ .bind(0)
+ .bind(&result.metadata);
+
+ query
+ .execute(&mut *self.transaction)
+ .await
+ .with_context(|| format!("Failed to execute721Token query: {:?}", result))?;
+
+ Ok(())
+ }
+}
diff --git a/crates/torii/core/src/executor.rs b/crates/torii/core/src/executor/mod.rs
similarity index 64%
rename from crates/torii/core/src/executor.rs
rename to crates/torii/core/src/executor/mod.rs
index a8c0a19c70..b2c3bf7093 100644
--- a/crates/torii/core/src/executor.rs
+++ b/crates/torii/core/src/executor/mod.rs
@@ -1,29 +1,34 @@
use std::collections::HashMap;
use std::mem;
use std::str::FromStr;
+use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use anyhow::{Context, Result};
+use cainome::cairo_serde::{ByteArray, CairoSerde};
use dojo_types::schema::{Struct, Ty};
-use sqlx::query::Query;
-use sqlx::sqlite::SqliteArguments;
use sqlx::{FromRow, Pool, Sqlite, Transaction};
-use starknet::core::types::{Felt, U256};
+use starknet::core::types::{BlockId, BlockTag, Felt, FunctionCall};
+use starknet::core::utils::{get_selector_from_name, parse_cairo_short_string};
+use starknet::providers::Provider;
use tokio::sync::broadcast::{Receiver, Sender};
use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
-use tokio::sync::oneshot;
+use tokio::sync::{oneshot, Semaphore};
+use tokio::task::JoinSet;
use tokio::time::Instant;
use tracing::{debug, error};
use crate::simple_broker::SimpleBroker;
-use crate::sql::utils::{felt_to_sql_string, sql_string_to_u256, u256_to_sql_string, I256};
-use crate::sql::FELT_DELIMITER;
+use crate::sql::utils::{felt_to_sql_string, I256};
use crate::types::{
ContractCursor, ContractType, Entity as EntityUpdated, Event as EventEmitted,
EventMessage as EventMessageUpdated, Model as ModelRegistered, OptimisticEntity,
OptimisticEventMessage,
};
+pub mod erc;
+pub use erc::{RegisterErc20TokenQuery, RegisterErc721TokenMetadata, RegisterErc721TokenQuery};
+
pub(crate) const LOG_TARGET: &str = "torii_core::executor";
#[derive(Debug, Clone)]
@@ -102,14 +107,19 @@ pub enum QueryType {
DeleteEntity(DeleteEntityQuery),
EventMessage(EventMessageQuery),
ApplyBalanceDiff(ApplyBalanceDiffQuery),
+ RegisterErc721Token(RegisterErc721TokenQuery),
+ RegisterErc20Token(RegisterErc20TokenQuery),
+ TokenTransfer,
RegisterModel,
StoreEvent,
+ // similar to execute but doesn't create a new transaction
+ Flush,
Execute,
Other,
}
#[derive(Debug)]
-pub struct Executor<'c> {
+pub struct Executor<'c, P: Provider + Sync + Send + 'static> {
// Queries should use `transaction` instead of `pool`
// This `pool` is only used to create a new `transaction`
pool: Pool<Sqlite>,
@@ -117,6 +127,16 @@ pub struct Executor<'c> {
publish_queue: Vec<BrokerMessage>,
rx: UnboundedReceiver<QueryMessage>,
shutdown_rx: Receiver<()>,
+ // These tasks are spawned to fetch ERC721 token metadata from the chain
+ // to not block the main loop
+ register_tasks: JoinSet<Result<RegisterErc721TokenMetadata>>,
+ // Some queries depends on the metadata being registered, so we defer them
+ // until the metadata is fetched
+ deferred_query_messages: Vec<QueryMessage>,
+ // It is used to make RPC calls to fetch token_uri data for erc721 contracts
+ provider: Arc<P>,
+ // Used to limit number of tasks that run in parallel to fetch metadata
+ semaphore: Arc<Semaphore>,
}
#[derive(Debug)]
@@ -174,19 +194,48 @@ impl QueryMessage {
rx,
)
}
+
+ pub fn flush_recv() -> (Self, oneshot::Receiver<Result<()>>) {
+ let (tx, rx) = oneshot::channel();
+ (
+ Self {
+ statement: "".to_string(),
+ arguments: vec![],
+ query_type: QueryType::Flush,
+ tx: Some(tx),
+ },
+ rx,
+ )
+ }
}
-impl<'c> Executor<'c> {
+impl<'c, P: Provider + Sync + Send + 'static> Executor<'c, P> {
pub async fn new(
pool: Pool<Sqlite>,
shutdown_tx: Sender<()>,
+ provider: Arc<P>,
+ max_concurrent_tasks: usize,
) -> Result<(Self, UnboundedSender<QueryMessage>)> {
let (tx, rx) = unbounded_channel();
let transaction = pool.begin().await?;
let publish_queue = Vec::new();
let shutdown_rx = shutdown_tx.subscribe();
-
- Ok((Executor { pool, transaction, publish_queue, rx, shutdown_rx }, tx))
+ let semaphore = Arc::new(Semaphore::new(max_concurrent_tasks));
+
+ Ok((
+ Executor {
+ pool,
+ transaction,
+ publish_queue,
+ rx,
+ shutdown_rx,
+ register_tasks: JoinSet::new(),
+ deferred_query_messages: Vec::new(),
+ provider,
+ semaphore,
+ },
+ tx,
+ ))
}
pub async fn run(&mut self) -> Result<()> {
@@ -197,41 +246,38 @@ impl<'c> Executor<'c> {
break Ok(());
}
Some(msg) = self.rx.recv() => {
- let QueryMessage { statement, arguments, query_type, tx } = msg;
- let mut query = sqlx::query(&statement);
-
- for arg in &arguments {
- query = match arg {
- Argument::Null => query.bind(None::<String>),
- Argument::Int(integer) => query.bind(integer),
- Argument::Bool(bool) => query.bind(bool),
- Argument::String(string) => query.bind(string),
- Argument::FieldElement(felt) => query.bind(format!("{:#x}", felt)),
- }
- }
-
- match self.handle_query_type(query, query_type.clone(), &statement, &arguments, tx).await {
+ let query_type = msg.query_type.clone();
+ match self.handle_query_message(msg).await {
Ok(()) => {},
Err(e) => {
error!(target: LOG_TARGET, r#type = ?query_type, error = %e, "Failed to execute query.");
}
}
}
+ Some(result) = self.register_tasks.join_next() => {
+ let result = result??;
+ self.handle_erc721_token_metadata(result).await?;
+ }
}
}
}
- async fn handle_query_type<'a>(
- &mut self,
- query: Query<'a, Sqlite, SqliteArguments<'a>>,
- query_type: QueryType,
- statement: &str,
- arguments: &[Argument],
- sender: Option<oneshot::Sender<Result<()>>>,
- ) -> Result<()> {
+ async fn handle_query_message(&mut self, query_message: QueryMessage) -> Result<()> {
let tx = &mut self.transaction;
- match query_type {
+ let mut query = sqlx::query(&query_message.statement);
+
+ for arg in &query_message.arguments {
+ query = match arg {
+ Argument::Null => query.bind(None::<String>),
+ Argument::Int(integer) => query.bind(integer),
+ Argument::Bool(bool) => query.bind(bool),
+ Argument::String(string) => query.bind(string),
+ Argument::FieldElement(felt) => query.bind(format!("{:#x}", felt)),
+ }
+ }
+
+ match query_message.query_type {
QueryType::SetHead(set_head) => {
let previous_block_timestamp: u64 = sqlx::query_scalar::<_, i64>(
"SELECT last_block_timestamp FROM contracts WHERE id = ?",
@@ -249,7 +295,10 @@ impl<'c> Executor<'c> {
};
query.execute(&mut **tx).await.with_context(|| {
- format!("Failed to execute query: {:?}, args: {:?}", statement, arguments)
+ format!(
+ "Failed to execute query: {:?}, args: {:?}",
+ query_message.statement, query_message.arguments
+ )
})?;
let row = sqlx::query("UPDATE contracts SET tps = ? WHERE id = ? RETURNING *")
@@ -373,7 +422,10 @@ impl<'c> Executor<'c> {
}
QueryType::SetEntity(entity) => {
let row = query.fetch_one(&mut **tx).await.with_context(|| {
- format!("Failed to execute query: {:?}, args: {:?}", statement, arguments)
+ format!(
+ "Failed to execute query: {:?}, args: {:?}",
+ query_message.statement, query_message.arguments
+ )
})?;
let mut entity_updated = EntityUpdated::from_row(&row)?;
entity_updated.updated_model = Some(entity);
@@ -396,7 +448,10 @@ impl<'c> Executor<'c> {
}
QueryType::DeleteEntity(entity) => {
let delete_model = query.execute(&mut **tx).await.with_context(|| {
- format!("Failed to execute query: {:?}, args: {:?}", statement, arguments)
+ format!(
+ "Failed to execute query: {:?}, args: {:?}",
+ query_message.statement, query_message.arguments
+ )
})?;
if delete_model.rows_affected() == 0 {
return Ok(());
@@ -447,7 +502,10 @@ impl<'c> Executor<'c> {
}
QueryType::RegisterModel => {
let row = query.fetch_one(&mut **tx).await.with_context(|| {
- format!("Failed to execute query: {:?}, args: {:?}", statement, arguments)
+ format!(
+ "Failed to execute query: {:?}, args: {:?}",
+ query_message.statement, query_message.arguments
+ )
})?;
let model_registered = ModelRegistered::from_row(&row)?;
self.publish_queue.push(BrokerMessage::ModelRegistered(model_registered));
@@ -455,7 +513,10 @@ impl<'c> Executor<'c> {
QueryType::EventMessage(em_query) => {
// Must be executed first since other tables have foreign keys on event_messages.id.
let event_messages_row = query.fetch_one(&mut **tx).await.with_context(|| {
- format!("Failed to execute query: {:?}, args: {:?}", statement, arguments)
+ format!(
+ "Failed to execute query: {:?}, args: {:?}",
+ query_message.statement, query_message.arguments
+ )
})?;
let mut event_counter: i64 = sqlx::query_scalar::<_, i64>(
@@ -525,7 +586,10 @@ impl<'c> Executor<'c> {
}
QueryType::StoreEvent => {
let row = query.fetch_one(&mut **tx).await.with_context(|| {
- format!("Failed to execute query: {:?}, args: {:?}", statement, arguments)
+ format!(
+ "Failed to execute query: {:?}, args: {:?}",
+ query_message.statement, query_message.arguments
+ )
})?;
let event = EventEmitted::from_row(&row)?;
self.publish_queue.push(BrokerMessage::EventEmitted(event));
@@ -536,13 +600,127 @@ impl<'c> Executor<'c> {
self.apply_balance_diff(apply_balance_diff).await?;
debug!(target: LOG_TARGET, duration = ?instant.elapsed(), "Applied balance diff.");
}
+ QueryType::RegisterErc721Token(register_erc721_token) => {
+ let semaphore = self.semaphore.clone();
+ let provider = self.provider.clone();
+ let res = sqlx::query_as::<_, (String, String)>(
+ "SELECT name, symbol FROM tokens WHERE contract_address = ?",
+ )
+ .bind(felt_to_sql_string(®ister_erc721_token.contract_address))
+ .fetch_one(&mut **tx)
+ .await;
+
+ // If we find a token already registered for this contract_address we dont need to
+ // refetch the data since its same for all ERC721 tokens
+ let (name, symbol) = match res {
+ Ok((name, symbol)) => {
+ debug!(
+ contract_address = %felt_to_sql_string(®ister_erc721_token.contract_address),
+ "Token already registered for contract_address, so reusing fetched data",
+ );
+ (name, symbol)
+ }
+ Err(_) => {
+ // Fetch token information from the chain
+ let name = provider
+ .call(
+ FunctionCall {
+ contract_address: register_erc721_token.contract_address,
+ entry_point_selector: get_selector_from_name("name").unwrap(),
+ calldata: vec![],
+ },
+ BlockId::Tag(BlockTag::Pending),
+ )
+ .await?;
+
+ // len = 1 => return value felt (i.e. legacy erc721 token)
+ // len > 1 => return value ByteArray (i.e. new erc721 token)
+ let name = if name.len() == 1 {
+ parse_cairo_short_string(&name[0]).unwrap()
+ } else {
+ ByteArray::cairo_deserialize(&name, 0)
+ .expect("Return value not ByteArray")
+ .to_string()
+ .expect("Return value not String")
+ };
+
+ let symbol = provider
+ .call(
+ FunctionCall {
+ contract_address: register_erc721_token.contract_address,
+ entry_point_selector: get_selector_from_name("symbol").unwrap(),
+ calldata: vec![],
+ },
+ BlockId::Tag(BlockTag::Pending),
+ )
+ .await?;
+ let symbol = if symbol.len() == 1 {
+ parse_cairo_short_string(&symbol[0]).unwrap()
+ } else {
+ ByteArray::cairo_deserialize(&symbol, 0)
+ .expect("Return value not ByteArray")
+ .to_string()
+ .expect("Return value not String")
+ };
+
+ (name, symbol)
+ }
+ };
+
+ self.register_tasks.spawn(async move {
+ let permit = semaphore.acquire().await.unwrap();
+
+ let result = Self::process_register_erc721_token_query(
+ register_erc721_token,
+ provider,
+ name,
+ symbol,
+ )
+ .await;
+
+ drop(permit);
+ result
+ });
+ }
+ QueryType::RegisterErc20Token(register_erc20_token) => {
+ let query = sqlx::query(
+ "INSERT INTO tokens (id, contract_address, name, symbol, decimals) VALUES (?, \
+ ?, ?, ?, ?)",
+ )
+ .bind(®ister_erc20_token.token_id)
+ .bind(felt_to_sql_string(®ister_erc20_token.contract_address))
+ .bind(®ister_erc20_token.name)
+ .bind(®ister_erc20_token.symbol)
+ .bind(register_erc20_token.decimals);
+
+ query.execute(&mut **tx).await.with_context(|| {
+ format!(
+ "Failed to execute RegisterErc20Token query: {:?}",
+ register_erc20_token
+ )
+ })?;
+ }
+ QueryType::Flush => {
+ debug!(target: LOG_TARGET, "Flushing query.");
+ let instant = Instant::now();
+ let res = self.execute(false).await;
+ debug!(target: LOG_TARGET, duration = ?instant.elapsed(), "Flushed query.");
+
+ if let Some(sender) = query_message.tx {
+ sender
+ .send(res)
+ .map_err(|_| anyhow::anyhow!("Failed to send execute result"))?;
+ } else {
+ res?;
+ }
+ }
QueryType::Execute => {
debug!(target: LOG_TARGET, "Executing query.");
let instant = Instant::now();
- let res = self.execute().await;
+ let res = self.execute(true).await;
debug!(target: LOG_TARGET, duration = ?instant.elapsed(), "Executed query.");
- if let Some(sender) = sender {
+ if let Some(sender) = query_message.tx {
sender
.send(res)
.map_err(|_| anyhow::anyhow!("Failed to send execute result"))?;
@@ -550,9 +728,16 @@ impl<'c> Executor<'c> {
res?;
}
}
+ QueryType::TokenTransfer => {
+ // defer executing these queries since they depend on TokenRegister queries
+ self.deferred_query_messages.push(query_message);
+ }
QueryType::Other => {
query.execute(&mut **tx).await.with_context(|| {
- format!("Failed to execute query: {:?}, args: {:?}", statement, arguments)
+ format!(
+ "Failed to execute query: {:?}, args: {:?}",
+ query_message.statement, query_message.arguments
+ )
})?;
}
}
@@ -560,109 +745,42 @@ impl<'c> Executor<'c> {
Ok(())
}
- async fn execute(&mut self) -> Result<()> {
- let transaction = mem::replace(&mut self.transaction, self.pool.begin().await?);
- transaction.commit().await?;
+ async fn execute(&mut self, new_transaction: bool) -> Result<()> {
+ if new_transaction {
+ let transaction = mem::replace(&mut self.transaction, self.pool.begin().await?);
+ transaction.commit().await?;
+ }
for message in self.publish_queue.drain(..) {
send_broker_message(message);
}
- Ok(())
- }
-
- async fn apply_balance_diff(
- &mut self,
- apply_balance_diff: ApplyBalanceDiffQuery,
- ) -> Result<()> {
- let erc_cache = apply_balance_diff.erc_cache;
- for ((contract_type, id_str), balance) in erc_cache.iter() {
- let id = id_str.split(FELT_DELIMITER).collect::<Vec<&str>>();
- match contract_type {
- ContractType::WORLD => unreachable!(),
- ContractType::ERC721 => {
- // account_address/contract_address:id => ERC721
- assert!(id.len() == 2);
- let account_address = id[0];
- let token_id = id[1];
- let mid = token_id.split(":").collect::<Vec<&str>>();
- let contract_address = mid[0];
-
- self.apply_balance_diff_helper(
- id_str,
- account_address,
- contract_address,
- token_id,
- balance,
- )
- .await
- .with_context(|| "Failed to apply balance diff in apply_cache_diff")?;
- }
- ContractType::ERC20 => {
- // account_address/contract_address/ => ERC20
- assert!(id.len() == 3);
- let account_address = id[0];
- let contract_address = id[1];
- let token_id = id[1];
-
- self.apply_balance_diff_helper(
- id_str,
- account_address,
- contract_address,
- token_id,
- balance,
- )
- .await
- .with_context(|| "Failed to apply balance diff in apply_cache_diff")?;
- }
- }
+ while let Some(result) = self.register_tasks.join_next().await {
+ let result = result??;
+ self.handle_erc721_token_metadata(result).await?;
}
- Ok(())
- }
-
- async fn apply_balance_diff_helper(
- &mut self,
- id: &str,
- account_address: &str,
- contract_address: &str,
- token_id: &str,
- balance_diff: &I256,
- ) -> Result<()> {
- let tx = &mut self.transaction;
- let balance: Option<(String,)> =
- sqlx::query_as("SELECT balance FROM balances WHERE id = ?")
- .bind(id)
- .fetch_optional(&mut **tx)
- .await?;
-
- let mut balance = if let Some(balance) = balance {
- sql_string_to_u256(&balance.0)
- } else {
- U256::from(0u8)
- };
-
- if balance_diff.is_negative {
- if balance < balance_diff.value {
- dbg!(&balance_diff, balance, id);
+ let mut deferred_query_messages = mem::take(&mut self.deferred_query_messages);
+
+ for query_message in deferred_query_messages.drain(..) {
+ let mut query = sqlx::query(&query_message.statement);
+ for arg in &query_message.arguments {
+ query = match arg {
+ Argument::Null => query.bind(None::<String>),
+ Argument::Int(integer) => query.bind(integer),
+ Argument::Bool(bool) => query.bind(bool),
+ Argument::String(string) => query.bind(string),
+ Argument::FieldElement(felt) => query.bind(format!("{:#x}", felt)),
+ };
}
- balance -= balance_diff.value;
- } else {
- balance += balance_diff.value;
- }
- // write the new balance to the database
- sqlx::query(
- "INSERT OR REPLACE INTO balances (id, contract_address, account_address, token_id, \
- balance) VALUES (?, ?, ?, ?, ?)",
- )
- .bind(id)
- .bind(contract_address)
- .bind(account_address)
- .bind(token_id)
- .bind(u256_to_sql_string(&balance))
- .execute(&mut **tx)
- .await?;
+ query.execute(&mut *self.transaction).await.with_context(|| {
+ format!(
+ "Failed to execute query: {:?}, args: {:?}",
+ query_message.statement, query_message.arguments
+ )
+ })?;
+ }
Ok(())
}
diff --git a/crates/torii/core/src/processors/erc721_legacy_transfer.rs b/crates/torii/core/src/processors/erc721_legacy_transfer.rs
index b3fdcbbfe8..ebbf4d6b53 100644
--- a/crates/torii/core/src/processors/erc721_legacy_transfer.rs
+++ b/crates/torii/core/src/processors/erc721_legacy_transfer.rs
@@ -36,7 +36,7 @@ where
async fn process(
&self,
- world: &WorldContractReader<P>,
+ _world: &WorldContractReader<P>,
db: &mut Sql,
_block_number: u64,
block_timestamp: u64,
@@ -51,16 +51,8 @@ where
let token_id = U256Cainome::cairo_deserialize(&event.data, 2)?;
let token_id = U256::from_words(token_id.low, token_id.high);
- db.handle_erc721_transfer(
- token_address,
- from,
- to,
- token_id,
- world.provider(),
- block_timestamp,
- event_id,
- )
- .await?;
+ db.handle_erc721_transfer(token_address, from, to, token_id, block_timestamp, event_id)
+ .await?;
debug!(target: LOG_TARGET, from = ?from, to = ?to, token_id = ?token_id, "ERC721 Transfer");
Ok(())
diff --git a/crates/torii/core/src/processors/erc721_transfer.rs b/crates/torii/core/src/processors/erc721_transfer.rs
index 266ea18e51..a0f56479a9 100644
--- a/crates/torii/core/src/processors/erc721_transfer.rs
+++ b/crates/torii/core/src/processors/erc721_transfer.rs
@@ -36,7 +36,7 @@ where
async fn process(
&self,
- world: &WorldContractReader<P>,
+ _world: &WorldContractReader<P>,
db: &mut Sql,
_block_number: u64,
block_timestamp: u64,
@@ -51,16 +51,8 @@ where
let token_id = U256Cainome::cairo_deserialize(&event.keys, 3)?;
let token_id = U256::from_words(token_id.low, token_id.high);
- db.handle_erc721_transfer(
- token_address,
- from,
- to,
- token_id,
- world.provider(),
- block_timestamp,
- event_id,
- )
- .await?;
+ db.handle_erc721_transfer(token_address, from, to, token_id, block_timestamp, event_id)
+ .await?;
debug!(target: LOG_TARGET, from = ?from, to = ?to, token_id = ?token_id, "ERC721 Transfer");
Ok(())
diff --git a/crates/torii/core/src/processors/metadata_update.rs b/crates/torii/core/src/processors/metadata_update.rs
index 76a9f37c12..00cedd800a 100644
--- a/crates/torii/core/src/processors/metadata_update.rs
+++ b/crates/torii/core/src/processors/metadata_update.rs
@@ -1,5 +1,3 @@
-use std::time::Duration;
-
use anyhow::{Error, Result};
use async_trait::async_trait;
use base64::engine::general_purpose;
@@ -9,17 +7,13 @@ use dojo_world::config::WorldMetadata;
use dojo_world::contracts::abigen::world::Event as WorldEvent;
use dojo_world::contracts::world::WorldContractReader;
use dojo_world::uri::Uri;
-use reqwest::Client;
use starknet::core::types::{Event, Felt};
use starknet::providers::Provider;
-use tokio_util::bytes::Bytes;
use tracing::{error, info};
use super::{EventProcessor, EventProcessorConfig};
use crate::sql::Sql;
-
-const IPFS_URL: &str = "https://cartridge.infura-ipfs.io/ipfs/";
-const MAX_RETRY: u8 = 3;
+use crate::utils::{fetch_content_from_ipfs, MAX_RETRY};
pub(crate) const LOG_TARGET: &str = "torii_core::processors::metadata_update";
@@ -112,7 +106,7 @@ async fn metadata(uri_str: String) -> Result<(WorldMetadata, Option<String>, Opt
let uri = Uri::Ipfs(uri_str);
let cid = uri.cid().ok_or("Uri is malformed").map_err(Error::msg)?;
- let bytes = fetch_content(cid, MAX_RETRY).await?;
+ let bytes = fetch_content_from_ipfs(cid, MAX_RETRY).await?;
let metadata: WorldMetadata = serde_json::from_str(std::str::from_utf8(&bytes)?)?;
let icon_img = fetch_image(&metadata.icon_uri).await;
@@ -123,36 +117,10 @@ async fn metadata(uri_str: String) -> Result<(WorldMetadata, Option<String>, Opt
async fn fetch_image(image_uri: &Option<Uri>) -> Option<String> {
if let Some(uri) = image_uri {
- let data = fetch_content(uri.cid()?, MAX_RETRY).await.ok()?;
+ let data = fetch_content_from_ipfs(uri.cid()?, MAX_RETRY).await.ok()?;
let encoded = general_purpose::STANDARD.encode(data);
return Some(encoded);
}
None
}
-
-async fn fetch_content(cid: &str, mut retries: u8) -> Result<Bytes> {
- while retries > 0 {
- let response = Client::new().get(format!("{IPFS_URL}{}", cid)).send().await;
-
- match response {
- Ok(response) => return response.bytes().await.map_err(|e| e.into()),
- Err(e) => {
- retries -= 1;
- if retries > 0 {
- info!(
- target: LOG_TARGET,
- error = %e,
- "Fetch uri."
- );
- tokio::time::sleep(Duration::from_secs(3)).await;
- }
- }
- }
- }
-
- Err(Error::msg(format!(
- "Failed to pull data from IPFS after {} attempts, cid: {}",
- MAX_RETRY, cid
- )))
-}
diff --git a/crates/torii/core/src/sql/erc.rs b/crates/torii/core/src/sql/erc.rs
index f82eacb1a2..0a7acc643d 100644
--- a/crates/torii/core/src/sql/erc.rs
+++ b/crates/torii/core/src/sql/erc.rs
@@ -6,11 +6,13 @@ use cainome::cairo_serde::{ByteArray, CairoSerde};
use starknet::core::types::{BlockId, BlockTag, Felt, FunctionCall, U256};
use starknet::core::utils::{get_selector_from_name, parse_cairo_short_string};
use starknet::providers::Provider;
-use tracing::debug;
use super::utils::{u256_to_sql_string, I256};
use super::{Sql, FELT_DELIMITER};
-use crate::executor::{ApplyBalanceDiffQuery, Argument, QueryMessage, QueryType};
+use crate::executor::{
+ ApplyBalanceDiffQuery, Argument, QueryMessage, QueryType, RegisterErc20TokenQuery,
+ RegisterErc721TokenQuery,
+};
use crate::sql::utils::{felt_and_u256_to_sql_string, felt_to_sql_string, felts_to_sql_string};
use crate::types::ContractType;
use crate::utils::utc_dt_string_from_timestamp;
@@ -34,7 +36,6 @@ impl Sql {
if !token_exists {
self.register_erc20_token_metadata(contract_address, &token_id, provider).await?;
- self.execute().await.with_context(|| "Failed to execute in handle_erc20_transfer")?;
}
self.store_erc_transfer_event(
@@ -66,6 +67,7 @@ impl Sql {
}
if self.local_cache.erc_cache.len() >= 100000 {
+ self.flush().await.with_context(|| "Failed to flush in handle_erc20_transfer")?;
self.apply_cache_diff().await?;
}
@@ -73,23 +75,23 @@ impl Sql {
}
#[allow(clippy::too_many_arguments)]
- pub async fn handle_erc721_transfer<P: Provider + Sync>(
+ pub async fn handle_erc721_transfer(
&mut self,
contract_address: Felt,
from_address: Felt,
to_address: Felt,
token_id: U256,
- provider: &P,
block_timestamp: u64,
event_id: &str,
) -> Result<()> {
// contract_address:id
+ let actual_token_id = token_id;
let token_id = felt_and_u256_to_sql_string(&contract_address, &token_id);
let token_exists: bool = self.local_cache.contains_token_id(&token_id);
if !token_exists {
- self.register_erc721_token_metadata(contract_address, &token_id, provider).await?;
- self.execute().await?;
+ self.register_erc721_token_metadata(contract_address, &token_id, actual_token_id)
+ .await?;
}
self.store_erc_transfer_event(
@@ -126,6 +128,7 @@ impl Sql {
}
if self.local_cache.erc_cache.len() >= 100000 {
+ self.flush().await.with_context(|| "Failed to flush in handle_erc721_transfer")?;
self.apply_cache_diff().await?;
}
@@ -193,18 +196,16 @@ impl Sql {
.await?;
let decimals = u8::cairo_deserialize(&decimals, 0).expect("Return value not u8");
- // Insert the token into the tokens table
- self.executor.send(QueryMessage::other(
- "INSERT INTO tokens (id, contract_address, name, symbol, decimals) VALUES (?, ?, ?, \
- ?, ?)"
- .to_string(),
- vec![
- Argument::String(token_id.to_string()),
- Argument::FieldElement(contract_address),
- Argument::String(name),
- Argument::String(symbol),
- Argument::Int(decimals.into()),
- ],
+ self.executor.send(QueryMessage::new(
+ "".to_string(),
+ vec![],
+ QueryType::RegisterErc20Token(RegisterErc20TokenQuery {
+ token_id: token_id.to_string(),
+ contract_address,
+ name,
+ symbol,
+ decimals,
+ }),
))?;
self.local_cache.register_token_id(token_id.to_string());
@@ -212,100 +213,26 @@ impl Sql {
Ok(())
}
- async fn register_erc721_token_metadata<P: Provider + Sync>(
+ async fn register_erc721_token_metadata(
&mut self,
contract_address: Felt,
token_id: &str,
- provider: &P,
+ actual_token_id: U256,
) -> Result<()> {
- let res = sqlx::query_as::<_, (String, String, u8)>(
- "SELECT name, symbol, decimals FROM tokens WHERE contract_address = ?",
- )
- .bind(felt_to_sql_string(&contract_address))
- .fetch_one(&self.pool)
- .await;
-
- // If we find a token already registered for this contract_address we dont need to refetch
- // the data since its same for all ERC721 tokens
- if let Ok((name, symbol, decimals)) = res {
- debug!(
- contract_address = %felt_to_sql_string(&contract_address),
- "Token already registered for contract_address, so reusing fetched data",
- );
- self.executor.send(QueryMessage::other(
- "INSERT INTO tokens (id, contract_address, name, symbol, decimals) VALUES (?, ?, \
- ?, ?, ?)"
- .to_string(),
- vec![
- Argument::String(token_id.to_string()),
- Argument::FieldElement(contract_address),
- Argument::String(name),
- Argument::String(symbol),
- Argument::Int(decimals.into()),
- ],
- ))?;
- self.local_cache.register_token_id(token_id.to_string());
- return Ok(());
- }
-
- // Fetch token information from the chain
- let name = provider
- .call(
- FunctionCall {
- contract_address,
- entry_point_selector: get_selector_from_name("name").unwrap(),
- calldata: vec![],
- },
- BlockId::Tag(BlockTag::Pending),
- )
- .await?;
-
- // len = 1 => return value felt (i.e. legacy erc721 token)
- // len > 1 => return value ByteArray (i.e. new erc721 token)
- let name = if name.len() == 1 {
- parse_cairo_short_string(&name[0]).unwrap()
- } else {
- ByteArray::cairo_deserialize(&name, 0)
- .expect("Return value not ByteArray")
- .to_string()
- .expect("Return value not String")
- };
-
- let symbol = provider
- .call(
- FunctionCall {
- contract_address,
- entry_point_selector: get_selector_from_name("symbol").unwrap(),
- calldata: vec![],
- },
- BlockId::Tag(BlockTag::Pending),
- )
- .await?;
- let symbol = if symbol.len() == 1 {
- parse_cairo_short_string(&symbol[0]).unwrap()
- } else {
- ByteArray::cairo_deserialize(&symbol, 0)
- .expect("Return value not ByteArray")
- .to_string()
- .expect("Return value not String")
- };
-
- let decimals = 0;
-
- // Insert the token into the tokens table
- self.executor.send(QueryMessage::other(
- "INSERT INTO tokens (id, contract_address, name, symbol, decimals) VALUES (?, ?, ?, \
- ?, ?)"
- .to_string(),
- vec![
- Argument::String(token_id.to_string()),
- Argument::FieldElement(contract_address),
- Argument::String(name),
- Argument::String(symbol),
- Argument::Int(decimals.into()),
- ],
+ self.executor.send(QueryMessage::new(
+ "".to_string(),
+ vec![],
+ QueryType::RegisterErc721Token(RegisterErc721TokenQuery {
+ token_id: token_id.to_string(),
+ contract_address,
+ actual_token_id,
+ }),
))?;
+ // optimistically add the token_id to cache
+ // this cache is used while applying the cache diff
+ // so we need to make sure that all RegisterErc*Token queries
+ // are applied before the cache diff is applied
self.local_cache.register_token_id(token_id.to_string());
Ok(())
@@ -326,7 +253,7 @@ impl Sql {
to_address, amount, token_id, executed_at) VALUES (?, ?, ?, ?, ?, ?, \
?)";
- self.executor.send(QueryMessage::other(
+ self.executor.send(QueryMessage::new(
insert_query.to_string(),
vec![
Argument::String(event_id.to_string()),
@@ -337,6 +264,7 @@ impl Sql {
Argument::String(token_id.to_string()),
Argument::String(utc_dt_string_from_timestamp(block_timestamp)),
],
+ QueryType::TokenTransfer,
))?;
Ok(())
diff --git a/crates/torii/core/src/sql/mod.rs b/crates/torii/core/src/sql/mod.rs
index 8c6302447d..11eedf6a3c 100644
--- a/crates/torii/core/src/sql/mod.rs
+++ b/crates/torii/core/src/sql/mod.rs
@@ -1305,4 +1305,10 @@ impl Sql {
self.executor.send(execute)?;
recv.await?
}
+
+ pub async fn flush(&self) -> Result<()> {
+ let (flush, recv) = QueryMessage::flush_recv();
+ self.executor.send(flush)?;
+ recv.await?
+ }
}
diff --git a/crates/torii/core/src/sql/test.rs b/crates/torii/core/src/sql/test.rs
index dde4934ee1..7d79f3ba4a 100644
--- a/crates/torii/core/src/sql/test.rs
+++ b/crates/torii/core/src/sql/test.rs
@@ -120,7 +120,8 @@ async fn test_load_from_remote(sequencer: &RunnerCtx) {
sqlx::migrate!("../migrations").run(&pool).await.unwrap();
let (shutdown_tx, _) = broadcast::channel(1);
- let (mut executor, sender) = Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+ let (mut executor, sender) =
+ Executor::new(pool.clone(), shutdown_tx.clone(), Arc::clone(&provider), 100).await.unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
@@ -280,7 +281,8 @@ async fn test_load_from_remote_del(sequencer: &RunnerCtx) {
sqlx::migrate!("../migrations").run(&pool).await.unwrap();
let (shutdown_tx, _) = broadcast::channel(1);
- let (mut executor, sender) = Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+ let (mut executor, sender) =
+ Executor::new(pool.clone(), shutdown_tx.clone(), Arc::clone(&provider), 100).await.unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
@@ -295,7 +297,7 @@ async fn test_load_from_remote_del(sequencer: &RunnerCtx) {
.await
.unwrap();
- let _ = bootstrap_engine(world_reader, db.clone(), provider).await;
+ let _ = bootstrap_engine(world_reader, db.clone(), Arc::clone(&provider)).await.unwrap();
// TODO: seems that we don't delete the record after delete only values are zeroed?
assert_eq!(count_table("ns-PlayerConfig", &pool).await, 0);
@@ -368,7 +370,9 @@ async fn test_update_with_set_record(sequencer: &RunnerCtx) {
sqlx::migrate!("../migrations").run(&pool).await.unwrap();
let (shutdown_tx, _) = broadcast::channel(1);
- let (mut executor, sender) = Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+
+ let (mut executor, sender) =
+ Executor::new(pool.clone(), shutdown_tx.clone(), Arc::clone(&provider), 100).await.unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
diff --git a/crates/torii/core/src/utils.rs b/crates/torii/core/src/utils.rs
index 55f7ee563e..d66d294509 100644
--- a/crates/torii/core/src/utils.rs
+++ b/crates/torii/core/src/utils.rs
@@ -1,4 +1,19 @@
+use std::time::Duration;
+
+use anyhow::Result;
use chrono::{DateTime, Utc};
+use futures_util::TryStreamExt;
+use ipfs_api_backend_hyper::{IpfsApi, IpfsClient, TryFromUri};
+use tokio_util::bytes::Bytes;
+use tracing::info;
+
+// pub const IPFS_URL: &str = "https://cartridge.infura-ipfs.io/ipfs/";
+pub const IPFS_URL: &str = "https://ipfs.io/ipfs/";
+pub const MAX_RETRY: u8 = 3;
+
+pub const IPFS_CLIENT_URL: &str = "https://ipfs.infura.io:5001";
+pub const IPFS_USERNAME: &str = "2EBrzr7ZASQZKH32sl2xWauXPSA";
+pub const IPFS_PASSWORD: &str = "12290b883db9138a8ae3363b6739d220";
pub fn must_utc_datetime_from_timestamp(timestamp: u64) -> DateTime<Utc> {
let naive_dt = DateTime::from_timestamp(timestamp as i64, 0)
@@ -10,6 +25,31 @@ pub fn utc_dt_string_from_timestamp(timestamp: u64) -> String {
must_utc_datetime_from_timestamp(timestamp).to_rfc3339()
}
+pub async fn fetch_content_from_ipfs(cid: &str, mut retries: u8) -> Result<Bytes> {
+ let client =
+ IpfsClient::from_str(IPFS_CLIENT_URL)?.with_credentials(IPFS_USERNAME, IPFS_PASSWORD);
+ while retries > 0 {
+ let response = client.cat(cid).map_ok(|chunk| chunk.to_vec()).try_concat().await;
+ match response {
+ Ok(stream) => return Ok(Bytes::from(stream)),
+ Err(e) => {
+ retries -= 1;
+ if retries > 0 {
+ info!(
+ error = %e,
+ "Fetch uri."
+ );
+ tokio::time::sleep(Duration::from_secs(3)).await;
+ }
+ }
+ }
+ }
+
+ Err(anyhow::anyhow!(format!(
+ "Failed to pull data from IPFS after {} attempts, cid: {}",
+ MAX_RETRY, cid
+ )))
+}
// tests
#[cfg(test)]
mod tests {
diff --git a/crates/torii/graphql/src/tests/metadata_test.rs b/crates/torii/graphql/src/tests/metadata_test.rs
index 408731d569..b9cfab8a93 100644
--- a/crates/torii/graphql/src/tests/metadata_test.rs
+++ b/crates/torii/graphql/src/tests/metadata_test.rs
@@ -5,11 +5,14 @@ mod tests {
use dojo_world::config::{ProfileConfig, WorldMetadata};
use sqlx::SqlitePool;
use starknet::core::types::Felt;
+ use starknet::providers::jsonrpc::HttpTransport;
+ use starknet::providers::JsonRpcClient;
use tokio::sync::broadcast;
use torii_core::executor::Executor;
use torii_core::sql::cache::ModelCache;
use torii_core::sql::Sql;
use torii_core::types::{Contract, ContractType};
+ use url::Url;
use crate::schema::build_schema;
use crate::tests::{run_graphql_query, Connection, Content, Metadata as SqlMetadata, Social};
@@ -54,8 +57,12 @@ mod tests {
#[sqlx::test(migrations = "../migrations")]
async fn test_metadata(pool: SqlitePool) {
let (shutdown_tx, _) = broadcast::channel(1);
+ let url: Url = "https://www.example.com".parse().unwrap();
+ let provider = Arc::new(JsonRpcClient::new(HttpTransport::new(url)));
let (mut executor, sender) =
- Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+ Executor::new(pool.clone(), shutdown_tx.clone(), Arc::clone(&provider), 100)
+ .await
+ .unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
@@ -120,8 +127,12 @@ mod tests {
#[sqlx::test(migrations = "../migrations")]
async fn test_empty_content(pool: SqlitePool) {
let (shutdown_tx, _) = broadcast::channel(1);
+ let url: Url = "https://www.example.com".parse().unwrap();
+ let provider = Arc::new(JsonRpcClient::new(HttpTransport::new(url)));
let (mut executor, sender) =
- Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+ Executor::new(pool.clone(), shutdown_tx.clone(), Arc::clone(&provider), 100)
+ .await
+ .unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
diff --git a/crates/torii/graphql/src/tests/mod.rs b/crates/torii/graphql/src/tests/mod.rs
index d351651b6c..06d7afa747 100644
--- a/crates/torii/graphql/src/tests/mod.rs
+++ b/crates/torii/graphql/src/tests/mod.rs
@@ -343,7 +343,8 @@ pub async fn spinup_types_test(path: &str) -> Result<SqlitePool> {
let world = WorldContractReader::new(world_address, Arc::clone(&provider));
let (shutdown_tx, _) = broadcast::channel(1);
- let (mut executor, sender) = Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+ let (mut executor, sender) =
+ Executor::new(pool.clone(), shutdown_tx.clone(), Arc::clone(&provider), 100).await.unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
diff --git a/crates/torii/graphql/src/tests/subscription_test.rs b/crates/torii/graphql/src/tests/subscription_test.rs
index f1b6455b91..39146feddf 100644
--- a/crates/torii/graphql/src/tests/subscription_test.rs
+++ b/crates/torii/graphql/src/tests/subscription_test.rs
@@ -12,13 +12,17 @@ mod tests {
use serial_test::serial;
use sqlx::SqlitePool;
use starknet::core::types::Event;
+ use starknet::providers::jsonrpc::HttpTransport;
+ use starknet::providers::JsonRpcClient;
use starknet_crypto::{poseidon_hash_many, Felt};
use tokio::sync::{broadcast, mpsc};
use torii_core::executor::Executor;
use torii_core::sql::cache::ModelCache;
use torii_core::sql::utils::felts_to_sql_string;
use torii_core::sql::Sql;
+ use torii_core::types::ContractType;
use torii_core::types::{Contract, ContractType};
+ use url::Url;
use crate::tests::{model_fixtures, run_graphql_subscription};
use crate::utils;
@@ -27,8 +31,11 @@ mod tests {
#[serial]
async fn test_entity_subscription(pool: SqlitePool) {
let (shutdown_tx, _) = broadcast::channel(1);
+ // used to fetch token_uri data for erc721 tokens so pass dummy for the test
+ let url: Url = "https://www.example.com".parse().unwrap();
+ let provider = Arc::new(JsonRpcClient::new(HttpTransport::new(url)));
let (mut executor, sender) =
- Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+ Executor::new(pool.clone(), shutdown_tx.clone(), provider, 100).await.unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
@@ -177,8 +184,13 @@ mod tests {
#[serial]
async fn test_entity_subscription_with_id(pool: SqlitePool) {
let (shutdown_tx, _) = broadcast::channel(1);
+
+ // dummy provider since its required to query data for erc721 tokens
+ let url: Url = "https://www.example.com".parse().unwrap();
+ let provider = Arc::new(JsonRpcClient::new(HttpTransport::new(url)));
+
let (mut executor, sender) =
- Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+ Executor::new(pool.clone(), shutdown_tx.clone(), provider, 100).await.unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
@@ -307,8 +319,11 @@ mod tests {
#[serial]
async fn test_model_subscription(pool: SqlitePool) {
let (shutdown_tx, _) = broadcast::channel(1);
+
+ let url: Url = "https://www.example.com".parse().unwrap();
+ let provider = Arc::new(JsonRpcClient::new(HttpTransport::new(url)));
let (mut executor, sender) =
- Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+ Executor::new(pool.clone(), shutdown_tx.clone(), provider, 100).await.unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
@@ -388,8 +403,11 @@ mod tests {
#[serial]
async fn test_model_subscription_with_id(pool: SqlitePool) {
let (shutdown_tx, _) = broadcast::channel(1);
+
+ let url: Url = "https://www.example.com".parse().unwrap();
+ let provider = Arc::new(JsonRpcClient::new(HttpTransport::new(url)));
let (mut executor, sender) =
- Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+ Executor::new(pool.clone(), shutdown_tx.clone(), provider, 100).await.unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
@@ -470,8 +488,11 @@ mod tests {
#[serial]
async fn test_event_emitted(pool: SqlitePool) {
let (shutdown_tx, _) = broadcast::channel(1);
+
+ let url: Url = "https://www.example.com".parse().unwrap();
+ let provider = Arc::new(JsonRpcClient::new(HttpTransport::new(url)));
let (mut executor, sender) =
- Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+ Executor::new(pool.clone(), shutdown_tx.clone(), provider, 100).await.unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
diff --git a/crates/torii/grpc/src/server/tests/entities_test.rs b/crates/torii/grpc/src/server/tests/entities_test.rs
index b5f1e83e1c..031779da58 100644
--- a/crates/torii/grpc/src/server/tests/entities_test.rs
+++ b/crates/torii/grpc/src/server/tests/entities_test.rs
@@ -89,7 +89,9 @@ async fn test_entities_queries(sequencer: &RunnerCtx) {
TransactionWaiter::new(tx.transaction_hash, &provider).await.unwrap();
let (shutdown_tx, _) = broadcast::channel(1);
- let (mut executor, sender) = Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+
+ let (mut executor, sender) =
+ Executor::new(pool.clone(), shutdown_tx.clone(), Arc::clone(&provider), 100).await.unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
diff --git a/crates/torii/libp2p/Cargo.toml b/crates/torii/libp2p/Cargo.toml
index 5e356d81f7..05cf945115 100644
--- a/crates/torii/libp2p/Cargo.toml
+++ b/crates/torii/libp2p/Cargo.toml
@@ -20,8 +20,8 @@ dojo-types.workspace = true
dojo-world.workspace = true
indexmap.workspace = true
serde_json.workspace = true
-starknet.workspace = true
starknet-crypto.workspace = true
+starknet.workspace = true
thiserror.workspace = true
tracing.workspace = true
diff --git a/crates/torii/libp2p/src/tests.rs b/crates/torii/libp2p/src/tests.rs
index dc14d131fa..5dc6ae6bf0 100644
--- a/crates/torii/libp2p/src/tests.rs
+++ b/crates/torii/libp2p/src/tests.rs
@@ -569,13 +569,15 @@ mod test {
let sequencer = KatanaRunner::new().expect("Failed to create Katana sequencer");
- let provider = JsonRpcClient::new(HttpTransport::new(sequencer.url()));
+ let provider = Arc::new(JsonRpcClient::new(HttpTransport::new(sequencer.url())));
let account = sequencer.account_data(0);
let (shutdown_tx, _) = broadcast::channel(1);
let (mut executor, sender) =
- Executor::new(pool.clone(), shutdown_tx.clone()).await.unwrap();
+ Executor::new(pool.clone(), shutdown_tx.clone(), Arc::clone(&provider), 100)
+ .await
+ .unwrap();
tokio::spawn(async move {
executor.run().await.unwrap();
});
diff --git a/crates/torii/migrations/20241014085532_add_metadata_field.sql b/crates/torii/migrations/20241014085532_add_metadata_field.sql
new file mode 100644
index 0000000000..f6f81432c2
--- /dev/null
+++ b/crates/torii/migrations/20241014085532_add_metadata_field.sql
@@ -0,0 +1 @@
+ALTER TABLE tokens ADD COLUMN metadata TEXT;
\ No newline at end of file
diff --git a/crates/torii/server/Cargo.toml b/crates/torii/server/Cargo.toml
index 1ca82c911c..14d503748d 100644
--- a/crates/torii/server/Cargo.toml
+++ b/crates/torii/server/Cargo.toml
@@ -6,17 +6,26 @@ version.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
+anyhow.workspace = true
base64.workspace = true
-http.workspace = true
+camino.workspace = true
+data-url.workspace = true
http-body = "0.4.5"
-hyper.workspace = true
+http.workspace = true
hyper-reverse-proxy = { git = "https://github.com/tarrencev/hyper-reverse-proxy" }
+hyper.workspace = true
+image.workspace = true
indexmap.workspace = true
lazy_static.workspace = true
+mime_guess.workspace = true
+reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
-tokio.workspace = true
+sqlx.workspace = true
tokio-util = "0.7.7"
-tower.workspace = true
+tokio.workspace = true
+torii-core.workspace = true
tower-http.workspace = true
+tower.workspace = true
tracing.workspace = true
+warp.workspace = true
diff --git a/crates/torii/server/src/artifacts.rs b/crates/torii/server/src/artifacts.rs
new file mode 100644
index 0000000000..260d1e48d0
--- /dev/null
+++ b/crates/torii/server/src/artifacts.rs
@@ -0,0 +1,336 @@
+use std::future::Future;
+use std::io::Cursor;
+use std::net::SocketAddr;
+use std::str::FromStr;
+
+use anyhow::{Context, Result};
+use camino::Utf8PathBuf;
+use data_url::mime::Mime;
+use data_url::DataUrl;
+use image::{DynamicImage, ImageFormat};
+use reqwest::Client;
+use serde::{Deserialize, Serialize};
+use sqlx::{Pool, Sqlite};
+use tokio::fs;
+use tokio::fs::File;
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio::sync::broadcast::Receiver;
+use torii_core::utils::{fetch_content_from_ipfs, MAX_RETRY};
+use tracing::{debug, error, trace};
+use warp::http::Response;
+use warp::path::Tail;
+use warp::{reject, Filter};
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ImageQuery {
+ #[serde(alias = "h")]
+ height: Option<u32>,
+ #[serde(alias = "w")]
+ width: Option<u32>,
+}
+
+async fn serve_static_file(
+ path: Tail,
+ artifacts_dir: Utf8PathBuf,
+ pool: Pool<Sqlite>,
+ query: ImageQuery,
+) -> Result<impl warp::Reply, warp::Rejection> {
+ let path = path.as_str();
+
+ // Split the path and validate format
+ let parts: Vec<&str> = path.split('/').collect();
+
+ if parts.len() != 3 || parts[2] != "image" {
+ return Err(reject::not_found());
+ }
+
+ // Validate contract_address format
+ if !parts[0].starts_with("0x") {
+ return Err(reject::not_found());
+ }
+
+ // Validate token_id format
+ if !parts[1].starts_with("0x") {
+ return Err(reject::not_found());
+ }
+
+ let token_image_dir = artifacts_dir.join(parts[0]).join(parts[1]);
+
+ let token_id = format!("{}:{}", parts[0], parts[1]);
+ if !token_image_dir.exists() {
+ match fetch_and_process_image(&artifacts_dir, &token_id, pool)
+ .await
+ .context(format!("Failed to fetch and process image for token_id: {}", token_id))
+ {
+ Ok(path) => path,
+ Err(e) => {
+ error!(error = %e, "Failed to fetch and process image for token_id: {}", token_id);
+ return Err(warp::reject::not_found());
+ }
+ };
+ }
+ let file_name = match file_name_from_dir_and_query(token_image_dir, &query) {
+ Ok(file_name) => file_name,
+ Err(e) => {
+ error!(error = %e, "Failed to get file name from directory and query");
+ return Err(reject::not_found());
+ }
+ };
+
+ match File::open(&file_name).await {
+ Ok(mut file) => {
+ let mut contents = vec![];
+ if file.read_to_end(&mut contents).await.is_ok() {
+ let mime = mime_guess::from_path(&file_name).first_or_octet_stream().to_string();
+
+ Ok(Response::builder().header("content-type", mime).body(contents))
+ } else {
+ Err(reject::not_found())
+ }
+ }
+ Err(_) => Err(reject::not_found()),
+ }
+}
+
+fn file_name_from_dir_and_query(
+ token_image_dir: Utf8PathBuf,
+ query: &ImageQuery,
+) -> Result<Utf8PathBuf> {
+ let mut entries = std::fs::read_dir(&token_image_dir).ok().into_iter().flatten().flatten();
+
+ // Find the base image (without @medium or @small)
+ let base_image = entries
+ .find(|entry| {
+ entry
+ .file_name()
+ .to_str()
+ .map(|name| name.starts_with("image") && !name.contains('@'))
+ .unwrap_or(false)
+ })
+ .with_context(|| "Failed to find base image")?;
+
+ let base_filename = base_image.file_name();
+ let base_filename = base_filename.to_str().unwrap();
+ let base_ext = base_filename.split('.').last().unwrap();
+
+ let suffix = match (query.width, query.height) {
+ // If either dimension is <= 100px, use small version
+ (Some(w), _) if w <= 100 => "@small",
+ (_, Some(h)) if h <= 100 => "@small",
+ // If either dimension is <= 250px, use medium version
+ (Some(w), _) if w <= 250 => "@medium",
+ (_, Some(h)) if h <= 250 => "@medium",
+ // If no dimensions specified or larger than 250px, use original
+ _ => "",
+ };
+
+ let target_filename = format!("image{}.{}", suffix, base_ext);
+ Ok(token_image_dir.join(target_filename))
+}
+
+pub async fn new(
+ mut shutdown_rx: Receiver<()>,
+ static_dir: &Utf8PathBuf,
+ pool: Pool<Sqlite>,
+) -> Result<(SocketAddr, impl Future<Output = ()> + 'static), std::io::Error> {
+ let static_dir = static_dir.clone();
+
+ let routes = warp::get()
+ .and(warp::path("static"))
+ .and(warp::path::tail())
+ .and(warp::any().map(move || static_dir.clone()))
+ .and(warp::any().map(move || pool.clone()))
+ .and(warp::any().and(warp::query::<ImageQuery>()))
+ .and_then(serve_static_file);
+
+ Ok(warp::serve(routes).bind_with_graceful_shutdown(([127, 0, 0, 1], 0), async move {
+ shutdown_rx.recv().await.ok();
+ }))
+}
+
+async fn fetch_and_process_image(
+ artifacts_path: &Utf8PathBuf,
+ token_id: &str,
+ pool: Pool<Sqlite>,
+) -> anyhow::Result<String> {
+ let query = sqlx::query_as::<_, (String,)>("SELECT metadata FROM tokens WHERE id = ?")
+ .bind(token_id)
+ .fetch_one(&pool)
+ .await
+ .with_context(|| {
+ format!("Failed to fetch metadata from database for token_id: {}", token_id)
+ })?;
+
+ let metadata: serde_json::Value =
+ serde_json::from_str(&query.0).context("Failed to parse metadata")?;
+ let image_uri = metadata
+ .get("image")
+ .with_context(|| format!("Image URL not found in metadata for token_id: {}", token_id))?
+ .as_str()
+ .with_context(|| format!("Image field not a string for token_id: {}", token_id))?
+ .to_string();
+
+ let image_type = match image_uri {
+ uri if uri.starts_with("http") || uri.starts_with("https") => {
+ debug!(image_uri = %uri, "Fetching image from http/https URL");
+ // Fetch image from HTTP/HTTPS URL
+ let client = Client::new();
+ let response = client
+ .get(uri)
+ .send()
+ .await
+ .context("Failed to fetch image from URL")?
+ .bytes()
+ .await
+ .context("Failed to read image bytes from response")?;
+
+ // svg files typically start with <svg or <?xml
+ if response.starts_with(b"<svg") || response.starts_with(b"<?xml") {
+ ErcImageType::Svg(response.to_vec())
+ } else {
+ let format = image::guess_format(&response).with_context(|| {
+ format!("Unknown file format for token_id: {}, data: {:?}", token_id, &response)
+ })?;
+ ErcImageType::DynamicImage((
+ image::load_from_memory(&response)
+ .context("Failed to load image from bytes")?,
+ format,
+ ))
+ }
+ }
+ uri if uri.starts_with("ipfs") => {
+ debug!(image_uri = %uri, "Fetching image from IPFS");
+ let cid = uri.strip_prefix("ipfs://").unwrap();
+ let response = fetch_content_from_ipfs(cid, MAX_RETRY)
+ .await
+ .context("Failed to read image bytes from IPFS response")?;
+
+ if response.starts_with(b"<svg") || response.starts_with(b"<?xml") {
+ ErcImageType::Svg(response.to_vec())
+ } else {
+ let format = image::guess_format(&response).with_context(|| {
+ format!(
+ "Unknown file format for token_id: {}, cid: {}, data: {:?}",
+ token_id, cid, &response
+ )
+ })?;
+ ErcImageType::DynamicImage((
+ image::load_from_memory(&response)
+ .context("Failed to load image from bytes")?,
+ format,
+ ))
+ }
+ }
+ uri if uri.starts_with("data") => {
+ debug!("Parsing image from data URI");
+ trace!(data_uri = %uri);
+ // Parse and decode data URI
+ let data_url = DataUrl::process(&uri).context("Failed to parse data URI")?;
+
+ // Check if it's an SVG
+ if data_url.mime_type() == &Mime::from_str("image/svg+xml").unwrap() {
+ let decoded = data_url.decode_to_vec().context("Failed to decode data URI")?;
+ ErcImageType::Svg(decoded.0)
+ } else {
+ let decoded = data_url.decode_to_vec().context("Failed to decode data URI")?;
+ let format = image::guess_format(&decoded.0)
+ .with_context(|| format!("Unknown file format for token_id: {}", token_id))?;
+ ErcImageType::DynamicImage((
+ image::load_from_memory(&decoded.0)
+ .context("Failed to load image from bytes")?,
+ format,
+ ))
+ }
+ }
+ uri => {
+ return Err(anyhow::anyhow!("Unsupported URI scheme: {}", uri));
+ }
+ };
+
+ // Extract contract_address and token_id from token_id
+ let parts: Vec<&str> = token_id.split(':').collect();
+ if parts.len() != 2 {
+ return Err(anyhow::anyhow!("token_id must be in format contract_address:token_id"));
+ }
+ let contract_address = parts[0];
+ let token_id_part = parts[1];
+
+ let dir_path = artifacts_path.join(contract_address).join(token_id_part);
+
+ // Create directories if they don't exist
+ fs::create_dir_all(&dir_path)
+ .await
+ .context("Failed to create directories for image storage")?;
+
+ // Define base image name
+ let base_image_name = "image";
+
+ let relative_path = Utf8PathBuf::new().join(contract_address).join(token_id_part);
+
+ match image_type {
+ ErcImageType::DynamicImage((img, format)) => {
+ let format_ext = format.extensions_str()[0];
+
+ let target_sizes = [("medium", 250, 250), ("small", 100, 100)];
+
+ // Save original image
+ let original_file_name = format!("{}.{}", base_image_name, format_ext);
+ let original_file_path = dir_path.join(&original_file_name);
+ let mut file = fs::File::create(&original_file_path)
+ .await
+ .with_context(|| format!("Failed to create file: {:?}", original_file_path))?;
+ let encoded_image = encode_image_to_vec(&img, format)
+ .with_context(|| format!("Failed to encode image: {:?}", original_file_path))?;
+ file.write_all(&encoded_image).await.with_context(|| {
+ format!("Failed to write image to file: {:?}", original_file_path)
+ })?;
+
+ // Save resized images
+ for (label, max_width, max_height) in &target_sizes {
+ let resized_image = resize_image_to_fit(&img, *max_width, *max_height);
+ let file_name = format!("@{}.{}", label, format_ext);
+ let file_path = dir_path.join(format!("{}{}", base_image_name, file_name));
+ let mut file = fs::File::create(&file_path)
+ .await
+ .with_context(|| format!("Failed to create file: {:?}", file_path))?;
+ let encoded_image = encode_image_to_vec(&resized_image, format)
+ .context("Failed to encode image")?;
+ file.write_all(&encoded_image)
+ .await
+ .with_context(|| format!("Failed to write image to file: {:?}", file_path))?;
+ }
+
+ Ok(format!("{}/{}", relative_path, base_image_name))
+ }
+ ErcImageType::Svg(svg_data) => {
+ let file_name = format!("{}.svg", base_image_name);
+ let file_path = dir_path.join(&file_name);
+
+ // Save the SVG file
+ let mut file = File::create(&file_path)
+ .await
+ .with_context(|| format!("Failed to create file: {:?}", file_path))?;
+ file.write_all(&svg_data)
+ .await
+ .with_context(|| format!("Failed to write SVG to file: {:?}", file_path))?;
+
+ Ok(format!("{}/{}", relative_path, file_name))
+ }
+ }
+}
+
+fn resize_image_to_fit(image: &DynamicImage, max_width: u32, max_height: u32) -> DynamicImage {
+ image.resize_to_fill(max_width, max_height, image::imageops::FilterType::Lanczos3)
+}
+
+fn encode_image_to_vec(image: &DynamicImage, format: ImageFormat) -> Result<Vec<u8>> {
+ let mut buf = Vec::new();
+ image.write_to(&mut Cursor::new(&mut buf), format).with_context(|| "Failed to encode image")?;
+ Ok(buf)
+}
+
+#[derive(Debug)]
+pub enum ErcImageType {
+ DynamicImage((DynamicImage, ImageFormat)),
+ Svg(Vec<u8>),
+}
diff --git a/crates/torii/server/src/lib.rs b/crates/torii/server/src/lib.rs
index 44dcc92d61..621f66d155 100644
--- a/crates/torii/server/src/lib.rs
+++ b/crates/torii/server/src/lib.rs
@@ -1 +1,2 @@
+pub mod artifacts;
pub mod proxy;
diff --git a/crates/torii/server/src/proxy.rs b/crates/torii/server/src/proxy.rs
index 4c759e8b1a..a43fa71c28 100644
--- a/crates/torii/server/src/proxy.rs
+++ b/crates/torii/server/src/proxy.rs
@@ -58,6 +58,7 @@ pub struct Proxy {
addr: SocketAddr,
allowed_origins: Option<Vec<String>>,
grpc_addr: Option<SocketAddr>,
+ artifacts_addr: Option<SocketAddr>,
graphql_addr: Arc<RwLock<Option<SocketAddr>>>,
}
@@ -67,8 +68,15 @@ impl Proxy {
allowed_origins: Option<Vec<String>>,
grpc_addr: Option<SocketAddr>,
graphql_addr: Option<SocketAddr>,
+ artifacts_addr: Option<SocketAddr>,
) -> Self {
- Self { addr, allowed_origins, grpc_addr, graphql_addr: Arc::new(RwLock::new(graphql_addr)) }
+ Self {
+ addr,
+ allowed_origins,
+ grpc_addr,
+ graphql_addr: Arc::new(RwLock::new(graphql_addr)),
+ artifacts_addr,
+ }
}
pub async fn set_graphql_addr(&self, addr: SocketAddr) {
@@ -84,6 +92,7 @@ impl Proxy {
let allowed_origins = self.allowed_origins.clone();
let grpc_addr = self.grpc_addr;
let graphql_addr = self.graphql_addr.clone();
+ let artifacts_addr = self.artifacts_addr;
let make_svc = make_service_fn(move |conn: &AddrStream| {
let remote_addr = conn.remote_addr().ip();
@@ -125,7 +134,7 @@ impl Proxy {
let graphql_addr = graphql_addr_clone.clone();
async move {
let graphql_addr = graphql_addr.read().await;
- handle(remote_addr, grpc_addr, *graphql_addr, req).await
+ handle(remote_addr, grpc_addr, artifacts_addr, *graphql_addr, req).await
}
});
@@ -145,9 +154,32 @@ impl Proxy {
async fn handle(
client_ip: IpAddr,
grpc_addr: Option<SocketAddr>,
+ artifacts_addr: Option<SocketAddr>,
graphql_addr: Option<SocketAddr>,
req: Request<Body>,
) -> Result<Response<Body>, Infallible> {
+ if req.uri().path().starts_with("/static") {
+ if let Some(artifacts_addr) = artifacts_addr {
+ let artifacts_addr = format!("http://{}", artifacts_addr);
+
+ return match GRAPHQL_PROXY_CLIENT.call(client_ip, &artifacts_addr, req).await {
+ Ok(response) => Ok(response),
+ Err(_error) => {
+ error!("{:?}", _error);
+ Ok(Response::builder()
+ .status(StatusCode::INTERNAL_SERVER_ERROR)
+ .body(Body::empty())
+ .unwrap())
+ }
+ };
+ } else {
+ return Ok(Response::builder()
+ .status(StatusCode::NOT_FOUND)
+ .body(Body::empty())
+ .unwrap());
+ }
+ }
+
if req.uri().path().starts_with("/graphql") {
if let Some(graphql_addr) = graphql_addr {
let graphql_addr = format!("http://{}", graphql_addr);
From 2cbb18b6706b6548f6f54476521d68a8e6e3d960 Mon Sep 17 00:00:00 2001
From: lambda-0x <0xlambda@protonmail.com>
Date: Wed, 23 Oct 2024 16:55:41 +0530
Subject: [PATCH 3/9] feat(torii/graphql): cleanup names and add imagePath and
metadata to erc_token
commit-id:21a1c0a5
---
bin/torii/src/main.rs | 5 +-
crates/torii/core/src/constants.rs | 3 +
crates/torii/core/src/executor/mod.rs | 7 +-
crates/torii/core/src/lib.rs | 1 +
crates/torii/core/src/sql/cache.rs | 16 ++---
crates/torii/core/src/sql/erc.rs | 8 ++-
crates/torii/core/src/utils.rs | 1 -
crates/torii/graphql/src/constants.rs | 16 ++---
crates/torii/graphql/src/mapping.rs | 30 +++++---
.../torii/graphql/src/object/erc/erc_token.rs | 29 ++++++--
crates/torii/graphql/src/object/erc/mod.rs | 4 +-
.../erc/{erc_balance.rs => token_balance.rs} | 65 ++++++++++++-----
.../{erc_transfer.rs => token_transfer.rs} | 70 ++++++++++++++-----
crates/torii/graphql/src/schema.rs | 7 +-
..._rename_tokens_and_erc_balances_tables.sql | 3 +
crates/torii/server/src/artifacts.rs | 17 +++--
16 files changed, 193 insertions(+), 89 deletions(-)
create mode 100644 crates/torii/core/src/constants.rs
rename crates/torii/graphql/src/object/erc/{erc_balance.rs => token_balance.rs} (78%)
rename crates/torii/graphql/src/object/erc/{erc_transfer.rs => token_transfer.rs} (79%)
create mode 100644 crates/torii/migrations/20241018073644_rename_tokens_and_erc_balances_tables.sql
diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs
index 1d5b2f1322..c605d58079 100644
--- a/bin/torii/src/main.rs
+++ b/bin/torii/src/main.rs
@@ -232,9 +232,7 @@ async fn main() -> anyhow::Result<()> {
args.max_concurrent_tasks,
)
.await?;
- tokio::spawn(async move {
- executor.run().await.unwrap();
- });
+ let executor_handle = tokio::spawn(async move { executor.run().await });
let model_cache = Arc::new(ModelCache::new(pool.clone()));
let db = Sql::new(pool.clone(), sender.clone(), &args.indexing.contracts, model_cache.clone())
@@ -350,6 +348,7 @@ async fn main() -> anyhow::Result<()> {
tokio::select! {
res = engine_handle => res??,
+ res = executor_handle => res??,
res = proxy_server_handle => res??,
res = graphql_server_handle => res?,
res = grpc_server_handle => res??,
diff --git a/crates/torii/core/src/constants.rs b/crates/torii/core/src/constants.rs
new file mode 100644
index 0000000000..8248b09f7d
--- /dev/null
+++ b/crates/torii/core/src/constants.rs
@@ -0,0 +1,3 @@
+pub const TOKEN_BALANCE_TABLE: &str = "token_balances";
+pub const TOKEN_TRANSFER_TABLE: &str = "token_transfers";
+pub const TOKENS_TABLE: &str = "tokens";
diff --git a/crates/torii/core/src/executor/mod.rs b/crates/torii/core/src/executor/mod.rs
index b2c3bf7093..c823aa1255 100644
--- a/crates/torii/core/src/executor/mod.rs
+++ b/crates/torii/core/src/executor/mod.rs
@@ -18,6 +18,7 @@ use tokio::task::JoinSet;
use tokio::time::Instant;
use tracing::{debug, error};
+use crate::constants::TOKENS_TABLE;
use crate::simple_broker::SimpleBroker;
use crate::sql::utils::{felt_to_sql_string, I256};
use crate::types::{
@@ -603,9 +604,9 @@ impl<'c, P: Provider + Sync + Send + 'static> Executor<'c, P> {
QueryType::RegisterErc721Token(register_erc721_token) => {
let semaphore = self.semaphore.clone();
let provider = self.provider.clone();
- let res = sqlx::query_as::<_, (String, String)>(
- "SELECT name, symbol FROM tokens WHERE contract_address = ?",
- )
+ let res = sqlx::query_as::<_, (String, String)>(&format!(
+ "SELECT name, symbol FROM {TOKENS_TABLE} WHERE contract_address = ?"
+ ))
.bind(felt_to_sql_string(®ister_erc721_token.contract_address))
.fetch_one(&mut **tx)
.await;
diff --git a/crates/torii/core/src/lib.rs b/crates/torii/core/src/lib.rs
index 0615f98b4e..fbf9a1e14b 100644
--- a/crates/torii/core/src/lib.rs
+++ b/crates/torii/core/src/lib.rs
@@ -1,5 +1,6 @@
#![warn(unused_crate_dependencies)]
+pub mod constants;
pub mod engine;
pub mod error;
pub mod executor;
diff --git a/crates/torii/core/src/sql/cache.rs b/crates/torii/core/src/sql/cache.rs
index 23da95bd34..76ea4a0574 100644
--- a/crates/torii/core/src/sql/cache.rs
+++ b/crates/torii/core/src/sql/cache.rs
@@ -6,6 +6,7 @@ use sqlx::{Pool, Sqlite, SqlitePool};
use starknet_crypto::Felt;
use tokio::sync::RwLock;
+use crate::constants::TOKEN_BALANCE_TABLE;
use crate::error::{Error, ParseError, QueryError};
use crate::model::{parse_sql_model_members, SqlModelMember};
use crate::sql::utils::I256;
@@ -133,27 +134,24 @@ pub struct LocalCache {
impl Clone for LocalCache {
fn clone(&self) -> Self {
- Self { erc_cache: HashMap::new(), token_id_registry: HashSet::new() }
+ Self { erc_cache: HashMap::new(), token_id_registry: self.token_id_registry.clone() }
}
}
impl LocalCache {
pub async fn new(pool: Pool<Sqlite>) -> Self {
// read existing token_id's from balances table and cache them
- let token_id_registry: Vec<(String,)> = sqlx::query_as("SELECT token_id FROM balances")
- .fetch_all(&pool)
- .await
- .expect("Should be able to read token_id's from blances table");
+ let token_id_registry: Vec<(String,)> =
+ sqlx::query_as(&format!("SELECT token_id FROM {TOKEN_BALANCE_TABLE}"))
+ .fetch_all(&pool)
+ .await
+ .expect("Should be able to read token_id's from blances table");
let token_id_registry = token_id_registry.into_iter().map(|token_id| token_id.0).collect();
Self { erc_cache: HashMap::new(), token_id_registry }
}
- pub fn empty() -> Self {
- Self { erc_cache: HashMap::new(), token_id_registry: HashSet::new() }
- }
-
pub fn contains_token_id(&self, token_id: &str) -> bool {
self.token_id_registry.contains(token_id)
}
diff --git a/crates/torii/core/src/sql/erc.rs b/crates/torii/core/src/sql/erc.rs
index 0a7acc643d..cef58f281a 100644
--- a/crates/torii/core/src/sql/erc.rs
+++ b/crates/torii/core/src/sql/erc.rs
@@ -9,6 +9,7 @@ use starknet::providers::Provider;
use super::utils::{u256_to_sql_string, I256};
use super::{Sql, FELT_DELIMITER};
+use crate::constants::TOKEN_TRANSFER_TABLE;
use crate::executor::{
ApplyBalanceDiffQuery, Argument, QueryMessage, QueryType, RegisterErc20TokenQuery,
RegisterErc721TokenQuery,
@@ -249,9 +250,10 @@ impl Sql {
block_timestamp: u64,
event_id: &str,
) -> Result<()> {
- let insert_query = "INSERT INTO erc_transfers (id, contract_address, from_address, \
- to_address, amount, token_id, executed_at) VALUES (?, ?, ?, ?, ?, ?, \
- ?)";
+ let insert_query = format!(
+ "INSERT INTO {TOKEN_TRANSFER_TABLE} (id, contract_address, from_address, to_address, \
+ amount, token_id, executed_at) VALUES (?, ?, ?, ?, ?, ?, ?)"
+ );
self.executor.send(QueryMessage::new(
insert_query.to_string(),
diff --git a/crates/torii/core/src/utils.rs b/crates/torii/core/src/utils.rs
index d66d294509..2556caa8f0 100644
--- a/crates/torii/core/src/utils.rs
+++ b/crates/torii/core/src/utils.rs
@@ -7,7 +7,6 @@ use ipfs_api_backend_hyper::{IpfsApi, IpfsClient, TryFromUri};
use tokio_util::bytes::Bytes;
use tracing::info;
-// pub const IPFS_URL: &str = "https://cartridge.infura-ipfs.io/ipfs/";
pub const IPFS_URL: &str = "https://ipfs.io/ipfs/";
pub const MAX_RETRY: u8 = 3;
diff --git a/crates/torii/graphql/src/constants.rs b/crates/torii/graphql/src/constants.rs
index fc5a376f56..1e89a0bea5 100644
--- a/crates/torii/graphql/src/constants.rs
+++ b/crates/torii/graphql/src/constants.rs
@@ -9,8 +9,6 @@ pub const EVENT_MESSAGE_TABLE: &str = "event_messages";
pub const MODEL_TABLE: &str = "models";
pub const TRANSACTION_TABLE: &str = "transactions";
pub const METADATA_TABLE: &str = "metadata";
-pub const ERC_BALANCE_TABLE: &str = "balances";
-pub const ERC_TRANSFER_TABLE: &str = "erc_transfers";
pub const ID_COLUMN: &str = "id";
pub const EVENT_ID_COLUMN: &str = "event_id";
@@ -35,9 +33,10 @@ pub const QUERY_TYPE_NAME: &str = "World__Query";
pub const SUBSCRIPTION_TYPE_NAME: &str = "World__Subscription";
pub const MODEL_ORDER_TYPE_NAME: &str = "World__ModelOrder";
pub const MODEL_ORDER_FIELD_TYPE_NAME: &str = "World__ModelOrderField";
-pub const ERC_BALANCE_TYPE_NAME: &str = "ERC__Balance";
-pub const ERC_TRANSFER_TYPE_NAME: &str = "ERC__Transfer";
-pub const ERC_TOKEN_TYPE_NAME: &str = "ERC__Token";
+pub const TOKEN_BALANCE_TYPE_NAME: &str = "Token__Balance";
+pub const TOKEN_TRANSFER_TYPE_NAME: &str = "Token__Transfer";
+pub const TOKEN_TYPE_NAME: &str = "ERC__Token";
+pub const ERC721_METADATA_TYPE_NAME: &str = "ERC721__Metadata";
// objects' single and plural names
pub const ENTITY_NAMES: (&str, &str) = ("entity", "entities");
@@ -49,10 +48,11 @@ pub const CONTENT_NAMES: (&str, &str) = ("content", "contents");
pub const METADATA_NAMES: (&str, &str) = ("metadata", "metadatas");
pub const TRANSACTION_NAMES: (&str, &str) = ("transaction", "transactions");
pub const PAGE_INFO_NAMES: (&str, &str) = ("pageInfo", "");
+pub const TOKEN_NAME: (&str, &str) = ("token", "tokens");
+pub const TOKEN_BALANCE_NAME: (&str, &str) = ("", "tokenBalances");
+pub const TOKEN_TRANSFER_NAME: (&str, &str) = ("", "tokenTransfers");
-pub const ERC_BALANCE_NAME: (&str, &str) = ("ercBalance", "");
-pub const ERC_TOKEN_NAME: (&str, &str) = ("ercToken", "");
-pub const ERC_TRANSFER_NAME: (&str, &str) = ("ercTransfer", "");
+pub const ERC721_METADATA_NAME: (&str, &str) = ("erc721Metadata", "");
// misc
pub const ORDER_DIR_TYPE_NAME: &str = "OrderDirection";
diff --git a/crates/torii/graphql/src/mapping.rs b/crates/torii/graphql/src/mapping.rs
index 089d2a6b51..adb8b37bc1 100644
--- a/crates/torii/graphql/src/mapping.rs
+++ b/crates/torii/graphql/src/mapping.rs
@@ -4,7 +4,9 @@ use async_graphql::Name;
use dojo_types::primitive::Primitive;
use lazy_static::lazy_static;
-use crate::constants::{CONTENT_TYPE_NAME, ERC_TOKEN_TYPE_NAME, SOCIAL_TYPE_NAME};
+use crate::constants::{
+ CONTENT_TYPE_NAME, ERC721_METADATA_TYPE_NAME, SOCIAL_TYPE_NAME, TOKEN_TYPE_NAME,
+};
use crate::types::{GraphqlType, TypeData, TypeMapping};
lazy_static! {
@@ -145,27 +147,39 @@ lazy_static! {
),
]);
- pub static ref ERC_BALANCE_TYPE_MAPPING: TypeMapping = IndexMap::from([
+ pub static ref TOKEN_BALANCE_TYPE_MAPPING: TypeMapping = IndexMap::from([
(Name::new("balance"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("type"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("tokenMetadata"), TypeData::Simple(TypeRef::named_nn(ERC_TOKEN_TYPE_NAME))),
+ (Name::new("tokenMetadata"), TypeData::Simple(TypeRef::named_nn(TOKEN_TYPE_NAME))),
]);
- pub static ref ERC_TRANSFER_TYPE_MAPPING: TypeMapping = IndexMap::from([
+ pub static ref TOKEN_TRANSFER_TYPE_MAPPING: TypeMapping = IndexMap::from([
(Name::new("from"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("to"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("amount"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("type"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("executedAt"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("tokenMetadata"), TypeData::Simple(TypeRef::named_nn(ERC_TOKEN_TYPE_NAME))),
+ (Name::new("tokenMetadata"), TypeData::Simple(TypeRef::named_nn(TOKEN_TYPE_NAME))),
(Name::new("transactionHash"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
]);
- pub static ref ERC_TOKEN_TYPE_MAPPING: TypeMapping = IndexMap::from([
+ pub static ref TOKEN_TYPE_MAPPING: TypeMapping = IndexMap::from([
(Name::new("name"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("symbol"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("tokenId"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("decimals"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("contractAddress"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("decimals"), TypeData::Simple(TypeRef::named(TypeRef::STRING))),
+ (
+ Name::new("erc721"),
+ TypeData::Nested((TypeRef::named(ERC721_METADATA_TYPE_NAME), IndexMap::new()))
+ ),
+ ]);
+
+ pub static ref ERC721_METADATA_TYPE_MAPPING: TypeMapping = IndexMap::from([
+ (Name::new("tokenId"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("name"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("description"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("attributes"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("imagePath"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("metadata"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
]);
}
diff --git a/crates/torii/graphql/src/object/erc/erc_token.rs b/crates/torii/graphql/src/object/erc/erc_token.rs
index 14b8de7877..d1a312b932 100644
--- a/crates/torii/graphql/src/object/erc/erc_token.rs
+++ b/crates/torii/graphql/src/object/erc/erc_token.rs
@@ -1,5 +1,7 @@
-use crate::constants::{ERC_TOKEN_NAME, ERC_TOKEN_TYPE_NAME};
-use crate::mapping::ERC_TOKEN_TYPE_MAPPING;
+use crate::constants::{
+ ERC721_METADATA_NAME, ERC721_METADATA_TYPE_NAME, TOKEN_NAME, TOKEN_TYPE_NAME,
+};
+use crate::mapping::{ERC721_METADATA_TYPE_MAPPING, TOKEN_TYPE_MAPPING};
use crate::object::BasicObject;
use crate::types::TypeMapping;
@@ -8,14 +10,31 @@ pub struct ErcTokenObject;
impl BasicObject for ErcTokenObject {
fn name(&self) -> (&str, &str) {
- ERC_TOKEN_NAME
+ TOKEN_NAME
}
fn type_name(&self) -> &str {
- ERC_TOKEN_TYPE_NAME
+ TOKEN_TYPE_NAME
}
fn type_mapping(&self) -> &TypeMapping {
- &ERC_TOKEN_TYPE_MAPPING
+ &TOKEN_TYPE_MAPPING
+ }
+}
+
+#[derive(Debug)]
+pub struct Erc721MetadataObject;
+
+impl BasicObject for Erc721MetadataObject {
+ fn name(&self) -> (&str, &str) {
+ ERC721_METADATA_NAME
+ }
+
+ fn type_name(&self) -> &str {
+ ERC721_METADATA_TYPE_NAME
+ }
+
+ fn type_mapping(&self) -> &TypeMapping {
+ &ERC721_METADATA_TYPE_MAPPING
}
}
diff --git a/crates/torii/graphql/src/object/erc/mod.rs b/crates/torii/graphql/src/object/erc/mod.rs
index 3e85722cca..72d0eb1fab 100644
--- a/crates/torii/graphql/src/object/erc/mod.rs
+++ b/crates/torii/graphql/src/object/erc/mod.rs
@@ -1,9 +1,9 @@
use super::connection::cursor;
use crate::query::order::CursorDirection;
-pub mod erc_balance;
pub mod erc_token;
-pub mod erc_transfer;
+pub mod token_balance;
+pub mod token_transfer;
fn handle_cursor(
cursor: &str,
diff --git a/crates/torii/graphql/src/object/erc/erc_balance.rs b/crates/torii/graphql/src/object/erc/token_balance.rs
similarity index 78%
rename from crates/torii/graphql/src/object/erc/erc_balance.rs
rename to crates/torii/graphql/src/object/erc/token_balance.rs
index 848303e71a..44818d9eee 100644
--- a/crates/torii/graphql/src/object/erc/erc_balance.rs
+++ b/crates/torii/graphql/src/object/erc/token_balance.rs
@@ -6,14 +6,13 @@ use serde::Deserialize;
use sqlx::sqlite::SqliteRow;
use sqlx::{FromRow, Pool, Row, Sqlite, SqliteConnection};
use starknet_crypto::Felt;
+use torii_core::constants::TOKEN_BALANCE_TABLE;
use torii_core::sql::utils::felt_to_sql_string;
use tracing::warn;
use super::handle_cursor;
-use crate::constants::{
- DEFAULT_LIMIT, ERC_BALANCE_NAME, ERC_BALANCE_TABLE, ERC_BALANCE_TYPE_NAME, ID_COLUMN,
-};
-use crate::mapping::ERC_BALANCE_TYPE_MAPPING;
+use crate::constants::{DEFAULT_LIMIT, ID_COLUMN, TOKEN_BALANCE_NAME, TOKEN_BALANCE_TYPE_NAME};
+use crate::mapping::TOKEN_BALANCE_TYPE_MAPPING;
use crate::object::connection::page_info::PageInfoObject;
use crate::object::connection::{
connection_arguments, cursor, parse_connection_arguments, ConnectionArguments,
@@ -30,15 +29,15 @@ pub struct ErcBalanceObject;
impl BasicObject for ErcBalanceObject {
fn name(&self) -> (&str, &str) {
- ERC_BALANCE_NAME
+ TOKEN_BALANCE_NAME
}
fn type_name(&self) -> &str {
- ERC_BALANCE_TYPE_NAME
+ TOKEN_BALANCE_TYPE_NAME
}
fn type_mapping(&self) -> &TypeMapping {
- &ERC_BALANCE_TYPE_MAPPING
+ &TOKEN_BALANCE_TYPE_MAPPING
}
}
@@ -51,7 +50,7 @@ impl ResolvableObject for ErcBalanceObject {
);
let mut field = Field::new(
- self.name().0,
+ self.name().1,
TypeRef::named(format!("{}Connection", self.type_name())),
move |ctx| {
FieldFuture::new(async move {
@@ -69,12 +68,12 @@ impl ResolvableObject for ErcBalanceObject {
}];
let total_count =
- count_rows(&mut conn, ERC_BALANCE_TABLE, &None, &Some(filter)).await?;
+ count_rows(&mut conn, TOKEN_BALANCE_TABLE, &None, &Some(filter)).await?;
let (data, page_info) =
- fetch_erc_balances(&mut conn, address, &connection, total_count).await?;
+ fetch_token_balances(&mut conn, address, &connection, total_count).await?;
- let results = erc_balance_connection_output(&data, total_count, page_info)?;
+ let results = token_balances_connection_output(&data, total_count, page_info)?;
Ok(Some(Value::Object(results)))
})
@@ -87,18 +86,18 @@ impl ResolvableObject for ErcBalanceObject {
}
}
-async fn fetch_erc_balances(
+async fn fetch_token_balances(
conn: &mut SqliteConnection,
address: Felt,
connection: &ConnectionArguments,
total_count: i64,
) -> sqlx::Result<(Vec<SqliteRow>, PageInfo)> {
- let table_name = ERC_BALANCE_TABLE;
+ let table_name = TOKEN_BALANCE_TABLE;
let id_column = format!("b.{}", ID_COLUMN);
let mut query = format!(
"SELECT b.id, t.contract_address, t.name, t.symbol, t.decimals, b.balance, b.token_id, \
- c.contract_type
+ t.metadata, c.contract_type
FROM {table_name} b
JOIN tokens t ON b.token_id = t.id
JOIN contracts c ON t.contract_address = c.contract_address"
@@ -207,7 +206,7 @@ async fn fetch_erc_balances(
}
}
-fn erc_balance_connection_output(
+fn token_balances_connection_output(
data: &[SqliteRow],
total_count: i64,
page_info: PageInfo,
@@ -222,10 +221,9 @@ fn erc_balance_connection_output(
let token_metadata = Value::Object(ValueMapping::from([
(Name::new("name"), Value::String(row.name)),
(Name::new("symbol"), Value::String(row.symbol)),
- // for erc20 there is no token_id
- (Name::new("tokenId"), Value::Null),
(Name::new("decimals"), Value::String(row.decimals.to_string())),
(Name::new("contractAddress"), Value::String(row.contract_address.clone())),
+ (Name::new("erc721"), Value::Null),
]));
Value::Object(ValueMapping::from([
@@ -239,12 +237,42 @@ fn erc_balance_connection_output(
let token_id = row.token_id.split(':').collect::<Vec<&str>>();
assert!(token_id.len() == 2);
+ let metadata: serde_json::Value =
+ serde_json::from_str(&row.metadata).expect("metadata is always json");
+ let erc721_name =
+ metadata.get("name").map(|v| v.to_string().trim_matches('"').to_string());
+ let erc721_description = metadata
+ .get("description")
+ .map(|v| v.to_string().trim_matches('"').to_string());
+ let erc721_attributes =
+ metadata.get("attributes").map(|v| v.to_string().trim_matches('"').to_string());
+
+ let image_path = format!("{}/{}", token_id.join("/"), "image");
let token_metadata = Value::Object(ValueMapping::from([
(Name::new("contractAddress"), Value::String(row.contract_address.clone())),
(Name::new("name"), Value::String(row.name)),
(Name::new("symbol"), Value::String(row.symbol)),
- (Name::new("tokenId"), Value::String(token_id[1].to_string())),
(Name::new("decimals"), Value::String(row.decimals.to_string())),
+ (
+ Name::new("erc721"),
+ Value::Object(ValueMapping::from([
+ (Name::new("imagePath"), Value::String(image_path)),
+ (Name::new("tokenId"), Value::String(token_id[1].to_string())),
+ (Name::new("metadata"), Value::String(row.metadata)),
+ (
+ Name::new("name"),
+ erc721_name.map(Value::String).unwrap_or(Value::Null),
+ ),
+ (
+ Name::new("description"),
+ erc721_description.map(Value::String).unwrap_or(Value::Null),
+ ),
+ (
+ Name::new("attributes"),
+ erc721_attributes.map(Value::String).unwrap_or(Value::Null),
+ ),
+ ])),
+ ),
]));
Value::Object(ValueMapping::from([
@@ -291,4 +319,5 @@ struct BalanceQueryResultRaw {
pub token_id: String,
pub balance: String,
pub contract_type: String,
+ pub metadata: String,
}
diff --git a/crates/torii/graphql/src/object/erc/erc_transfer.rs b/crates/torii/graphql/src/object/erc/token_transfer.rs
similarity index 79%
rename from crates/torii/graphql/src/object/erc/erc_transfer.rs
rename to crates/torii/graphql/src/object/erc/token_transfer.rs
index e1c950db39..2dc8d8bd13 100644
--- a/crates/torii/graphql/src/object/erc/erc_transfer.rs
+++ b/crates/torii/graphql/src/object/erc/token_transfer.rs
@@ -6,15 +6,14 @@ use serde::Deserialize;
use sqlx::sqlite::SqliteRow;
use sqlx::{FromRow, Pool, Row, Sqlite, SqliteConnection};
use starknet_crypto::Felt;
+use torii_core::constants::TOKEN_TRANSFER_TABLE;
use torii_core::engine::get_transaction_hash_from_event_id;
use torii_core::sql::utils::felt_to_sql_string;
use tracing::warn;
use super::handle_cursor;
-use crate::constants::{
- DEFAULT_LIMIT, ERC_TRANSFER_NAME, ERC_TRANSFER_TABLE, ERC_TRANSFER_TYPE_NAME, ID_COLUMN,
-};
-use crate::mapping::ERC_TRANSFER_TYPE_MAPPING;
+use crate::constants::{DEFAULT_LIMIT, ID_COLUMN, TOKEN_TRANSFER_NAME, TOKEN_TRANSFER_TYPE_NAME};
+use crate::mapping::TOKEN_TRANSFER_TYPE_MAPPING;
use crate::object::connection::page_info::PageInfoObject;
use crate::object::connection::{
connection_arguments, cursor, parse_connection_arguments, ConnectionArguments,
@@ -29,15 +28,15 @@ pub struct ErcTransferObject;
impl BasicObject for ErcTransferObject {
fn name(&self) -> (&str, &str) {
- ERC_TRANSFER_NAME
+ TOKEN_TRANSFER_NAME
}
fn type_name(&self) -> &str {
- ERC_TRANSFER_TYPE_NAME
+ TOKEN_TRANSFER_TYPE_NAME
}
fn type_mapping(&self) -> &TypeMapping {
- &ERC_TRANSFER_TYPE_MAPPING
+ &TOKEN_TRANSFER_TYPE_MAPPING
}
}
@@ -50,7 +49,7 @@ impl ResolvableObject for ErcTransferObject {
);
let mut field = Field::new(
- self.name().0,
+ self.name().1,
TypeRef::named(format!("{}Connection", self.type_name())),
move |ctx| {
FieldFuture::new(async move {
@@ -61,10 +60,10 @@ impl ResolvableObject for ErcTransferObject {
&account_address.to_case(Case::Camel),
)?;
- let total_count: (i64,) = sqlx::query_as(
- "SELECT COUNT(*) FROM erc_transfers WHERE from_address = ? OR to_address \
- = ?",
- )
+ let total_count: (i64,) = sqlx::query_as(&format!(
+ "SELECT COUNT(*) FROM {TOKEN_TRANSFER_TABLE} WHERE from_address = ? OR \
+ to_address = ?"
+ ))
.bind(felt_to_sql_string(&address))
.bind(felt_to_sql_string(&address))
.fetch_one(&mut *conn)
@@ -72,8 +71,8 @@ impl ResolvableObject for ErcTransferObject {
let total_count = total_count.0;
let (data, page_info) =
- fetch_erc_transfers(&mut conn, address, &connection, total_count).await?;
- let results = erc_transfer_connection_output(&data, total_count, page_info)?;
+ fetch_token_transfers(&mut conn, address, &connection, total_count).await?;
+ let results = token_transfers_connection_output(&data, total_count, page_info)?;
Ok(Some(Value::Object(results)))
})
@@ -86,13 +85,13 @@ impl ResolvableObject for ErcTransferObject {
}
}
-async fn fetch_erc_transfers(
+async fn fetch_token_transfers(
conn: &mut SqliteConnection,
address: Felt,
connection: &ConnectionArguments,
total_count: i64,
) -> sqlx::Result<(Vec<SqliteRow>, PageInfo)> {
- let table_name = ERC_TRANSFER_TABLE;
+ let table_name = TOKEN_TRANSFER_TABLE;
let id_column = format!("et.{}", ID_COLUMN);
let mut query = format!(
@@ -108,7 +107,8 @@ SELECT
t.name,
t.symbol,
t.decimals,
- c.contract_type
+ c.contract_type,
+ t.metadata
FROM
{table_name} et
JOIN
@@ -227,7 +227,7 @@ JOIN
}
}
-fn erc_transfer_connection_output(
+fn token_transfers_connection_output(
data: &[SqliteRow],
total_count: i64,
page_info: PageInfo,
@@ -248,6 +248,7 @@ fn erc_transfer_connection_output(
(Name::new("tokenId"), Value::Null),
(Name::new("decimals"), Value::String(row.decimals.to_string())),
(Name::new("contractAddress"), Value::String(row.contract_address.clone())),
+ (Name::new("erc721"), Value::Null),
]));
Value::Object(ValueMapping::from([
@@ -265,12 +266,42 @@ fn erc_transfer_connection_output(
let token_id = row.token_id.split(':').collect::<Vec<&str>>();
assert!(token_id.len() == 2);
+ let image_path = format!("{}/{}", token_id.join("/"), "image");
+ let metadata: serde_json::Value =
+ serde_json::from_str(&row.metadata).expect("metadata is always json");
+ let erc721_name =
+ metadata.get("name").map(|v| v.to_string().trim_matches('"').to_string());
+ let erc721_description = metadata
+ .get("description")
+ .map(|v| v.to_string().trim_matches('"').to_string());
+ let erc721_attributes =
+ metadata.get("attributes").map(|v| v.to_string().trim_matches('"').to_string());
+
let token_metadata = Value::Object(ValueMapping::from([
(Name::new("name"), Value::String(row.name)),
(Name::new("symbol"), Value::String(row.symbol)),
- (Name::new("tokenId"), Value::String(token_id[1].to_string())),
(Name::new("decimals"), Value::String(row.decimals.to_string())),
(Name::new("contractAddress"), Value::String(row.contract_address.clone())),
+ (
+ Name::new("erc721"),
+ Value::Object(ValueMapping::from([
+ (Name::new("imagePath"), Value::String(image_path)),
+ (Name::new("tokenId"), Value::String(token_id[1].to_string())),
+ (Name::new("metadata"), Value::String(row.metadata)),
+ (
+ Name::new("name"),
+ erc721_name.map(Value::String).unwrap_or(Value::Null),
+ ),
+ (
+ Name::new("description"),
+ erc721_description.map(Value::String).unwrap_or(Value::Null),
+ ),
+ (
+ Name::new("attributes"),
+ erc721_attributes.map(Value::String).unwrap_or(Value::Null),
+ ),
+ ])),
+ ),
]));
Value::Object(ValueMapping::from([
@@ -324,4 +355,5 @@ struct TransferQueryResultRaw {
pub symbol: String,
pub decimals: u8,
pub contract_type: String,
+ pub metadata: String,
}
diff --git a/crates/torii/graphql/src/schema.rs b/crates/torii/graphql/src/schema.rs
index 5f70c49908..7d9b90b883 100644
--- a/crates/torii/graphql/src/schema.rs
+++ b/crates/torii/graphql/src/schema.rs
@@ -10,9 +10,9 @@ use super::object::model_data::ModelDataObject;
use super::types::ScalarType;
use super::utils;
use crate::constants::{QUERY_TYPE_NAME, SUBSCRIPTION_TYPE_NAME};
-use crate::object::erc::erc_balance::ErcBalanceObject;
-use crate::object::erc::erc_token::ErcTokenObject;
-use crate::object::erc::erc_transfer::ErcTransferObject;
+use crate::object::erc::erc_token::{Erc721MetadataObject, ErcTokenObject};
+use crate::object::erc::token_balance::ErcBalanceObject;
+use crate::object::erc::token_transfer::ErcTransferObject;
use crate::object::event_message::EventMessageObject;
use crate::object::metadata::content::ContentObject;
use crate::object::metadata::social::SocialObject;
@@ -122,6 +122,7 @@ async fn build_objects(pool: &SqlitePool) -> Result<(Vec<ObjectVariant>, Vec<Uni
ObjectVariant::Basic(Box::new(ContentObject)),
ObjectVariant::Basic(Box::new(PageInfoObject)),
ObjectVariant::Basic(Box::new(ErcTokenObject)),
+ ObjectVariant::Basic(Box::new(Erc721MetadataObject)),
];
// model union object
diff --git a/crates/torii/migrations/20241018073644_rename_tokens_and_erc_balances_tables.sql b/crates/torii/migrations/20241018073644_rename_tokens_and_erc_balances_tables.sql
new file mode 100644
index 0000000000..d3d95f80ac
--- /dev/null
+++ b/crates/torii/migrations/20241018073644_rename_tokens_and_erc_balances_tables.sql
@@ -0,0 +1,3 @@
+-- Add migration script here
+ALTER TABLE balances RENAME TO token_balances;
+ALTER TABLE erc_transfers RENAME TO token_transfers;
diff --git a/crates/torii/server/src/artifacts.rs b/crates/torii/server/src/artifacts.rs
index 260d1e48d0..f98a3e2108 100644
--- a/crates/torii/server/src/artifacts.rs
+++ b/crates/torii/server/src/artifacts.rs
@@ -15,6 +15,7 @@ use tokio::fs;
use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::sync::broadcast::Receiver;
+use torii_core::constants::TOKENS_TABLE;
use torii_core::utils::{fetch_content_from_ipfs, MAX_RETRY};
use tracing::{debug, error, trace};
use warp::http::Response;
@@ -153,13 +154,15 @@ async fn fetch_and_process_image(
token_id: &str,
pool: Pool<Sqlite>,
) -> anyhow::Result<String> {
- let query = sqlx::query_as::<_, (String,)>("SELECT metadata FROM tokens WHERE id = ?")
- .bind(token_id)
- .fetch_one(&pool)
- .await
- .with_context(|| {
- format!("Failed to fetch metadata from database for token_id: {}", token_id)
- })?;
+ let query = sqlx::query_as::<_, (String,)>(&format!(
+ "SELECT metadata FROM {TOKENS_TABLE} WHERE id = ?"
+ ))
+ .bind(token_id)
+ .fetch_one(&pool)
+ .await
+ .with_context(|| {
+ format!("Failed to fetch metadata from database for token_id: {}", token_id)
+ })?;
let metadata: serde_json::Value =
serde_json::from_str(&query.0).context("Failed to parse metadata")?;
From 17fd08424c319d896fcb360b5b0ae701873ce456 Mon Sep 17 00:00:00 2001
From: lambda-0x <0xlambda@protonmail.com>
Date: Wed, 30 Oct 2024 23:51:28 +0530
Subject: [PATCH 4/9] fix(torii/core): handle an edge case with pending block
processing
commit-id:6f679e25
---
crates/torii/core/src/engine.rs | 30 ++++++++++++++++++------------
1 file changed, 18 insertions(+), 12 deletions(-)
diff --git a/crates/torii/core/src/engine.rs b/crates/torii/core/src/engine.rs
index edbbea147f..36d7d788fb 100644
--- a/crates/torii/core/src/engine.rs
+++ b/crates/torii/core/src/engine.rs
@@ -10,9 +10,9 @@ use dojo_world::contracts::world::WorldContractReader;
use futures_util::future::{join_all, try_join_all};
use hashlink::LinkedHashMap;
use starknet::core::types::{
- BlockId, BlockTag, EmittedEvent, Event, EventFilter, EventsPage, MaybePendingBlockWithReceipts,
- MaybePendingBlockWithTxHashes, PendingBlockWithReceipts, Transaction, TransactionReceipt,
- TransactionWithReceipt,
+ BlockHashAndNumber, BlockId, BlockTag, EmittedEvent, Event, EventFilter, EventsPage,
+ MaybePendingBlockWithReceipts, MaybePendingBlockWithTxHashes, PendingBlockWithReceipts,
+ Transaction, TransactionReceipt, TransactionWithReceipt,
};
use starknet::core::utils::get_selector_from_name;
use starknet::providers::Provider;
@@ -309,23 +309,23 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
// TODO: since we now process blocks in chunks we can parallelize the fetching of data
pub async fn fetch_data(&mut self, cursors: &Cursors) -> Result<FetchDataResult> {
- let latest_block_number = self.provider.block_hash_and_number().await?.block_number;
+ let latest_block = self.provider.block_hash_and_number().await?;
let from = cursors.head.unwrap_or(0);
- let total_remaining_blocks = latest_block_number - from;
+ let total_remaining_blocks = latest_block.block_number - from;
let blocks_to_process = total_remaining_blocks.min(self.config.blocks_chunk_size);
let to = from + blocks_to_process;
let instant = Instant::now();
- let result = if from < latest_block_number {
+ let result = if from < latest_block.block_number {
let from = if from == 0 { from } else { from + 1 };
let data = self.fetch_range(from, to, &cursors.cursor_map).await?;
debug!(target: LOG_TARGET, duration = ?instant.elapsed(), from = %from, to = %to, "Fetched data for range.");
FetchDataResult::Range(data)
} else if self.config.index_pending {
let data =
- self.fetch_pending(latest_block_number + 1, cursors.last_pending_block_tx).await?;
- debug!(target: LOG_TARGET, duration = ?instant.elapsed(), latest_block_number = %latest_block_number, "Fetched pending data.");
+ self.fetch_pending(latest_block.clone(), cursors.last_pending_block_tx).await?;
+ debug!(target: LOG_TARGET, duration = ?instant.elapsed(), latest_block_number = %latest_block.block_number, "Fetched pending data.");
if let Some(data) = data {
FetchDataResult::Pending(data)
} else {
@@ -453,12 +453,18 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
async fn fetch_pending(
&self,
- block_number: u64,
+ block: BlockHashAndNumber,
last_pending_block_tx: Option<Felt>,
) -> Result<Option<FetchPendingResult>> {
- let block = if let MaybePendingBlockWithReceipts::PendingBlock(pending) =
+ let pending_block = if let MaybePendingBlockWithReceipts::PendingBlock(pending) =
self.provider.get_block_with_receipts(BlockId::Tag(BlockTag::Pending)).await?
{
+ // if the parent hash is not the hash of the latest block that we fetched, then it means
+ // a new block got mined just after we fetched the latest block information
+ if block.block_hash != pending.parent_hash {
+ return Ok(None);
+ }
+
pending
} else {
// TODO: change this to unreachable once katana is updated to return PendingBlockWithTxs
@@ -468,8 +474,8 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
};
Ok(Some(FetchPendingResult {
- pending_block: Box::new(block),
- block_number,
+ pending_block: Box::new(pending_block),
+ block_number: block.block_number + 1,
last_pending_block_tx,
}))
}
From 013f994524e2f94d3441f9de511d2ff903567119 Mon Sep 17 00:00:00 2001
From: lambda-0x <0xlambda@protonmail.com>
Date: Sun, 3 Nov 2024 20:38:47 +0530
Subject: [PATCH 5/9] feat(torii/graphql): use union types for token balances
and transfers
commit-id:73b9b2b4
---
Cargo.lock | 28 ----
crates/torii/graphql/src/constants.rs | 12 +-
crates/torii/graphql/src/mapping.rs | 35 ++---
.../torii/graphql/src/object/erc/erc_token.rs | 99 +++++++++++--
crates/torii/graphql/src/object/erc/mod.rs | 15 ++
.../graphql/src/object/erc/token_balance.rs | 104 +++++--------
.../graphql/src/object/erc/token_transfer.rs | 139 ++++++++----------
crates/torii/graphql/src/object/mod.rs | 135 ++++++++++++++++-
crates/torii/graphql/src/schema.rs | 16 +-
9 files changed, 368 insertions(+), 215 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index e4f92c298e..84744fddd2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -10099,34 +10099,6 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8"
-[[package]]
-name = "notify"
-version = "7.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009"
-dependencies = [
- "bitflags 2.6.0",
- "filetime",
- "fsevent-sys",
- "inotify",
- "kqueue",
- "libc",
- "log",
- "mio",
- "notify-types",
- "walkdir",
- "windows-sys 0.52.0",
-]
-
-[[package]]
-name = "notify-types"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7393c226621f817964ffb3dc5704f9509e107a8b024b489cc2c1b217378785df"
-dependencies = [
- "instant",
-]
-
[[package]]
name = "ntapi"
version = "0.4.1"
diff --git a/crates/torii/graphql/src/constants.rs b/crates/torii/graphql/src/constants.rs
index 1e89a0bea5..f309b82b91 100644
--- a/crates/torii/graphql/src/constants.rs
+++ b/crates/torii/graphql/src/constants.rs
@@ -36,7 +36,10 @@ pub const MODEL_ORDER_FIELD_TYPE_NAME: &str = "World__ModelOrderField";
pub const TOKEN_BALANCE_TYPE_NAME: &str = "Token__Balance";
pub const TOKEN_TRANSFER_TYPE_NAME: &str = "Token__Transfer";
pub const TOKEN_TYPE_NAME: &str = "ERC__Token";
-pub const ERC721_METADATA_TYPE_NAME: &str = "ERC721__Metadata";
+// pub const ERC721_METADATA_TYPE_NAME: &str = "ERC721__Metadata";
+
+pub const ERC20_TYPE_NAME: &str = "ERC20__Token";
+pub const ERC721_TYPE_NAME: &str = "ERC721__Token";
// objects' single and plural names
pub const ENTITY_NAMES: (&str, &str) = ("entity", "entities");
@@ -48,11 +51,14 @@ pub const CONTENT_NAMES: (&str, &str) = ("content", "contents");
pub const METADATA_NAMES: (&str, &str) = ("metadata", "metadatas");
pub const TRANSACTION_NAMES: (&str, &str) = ("transaction", "transactions");
pub const PAGE_INFO_NAMES: (&str, &str) = ("pageInfo", "");
-pub const TOKEN_NAME: (&str, &str) = ("token", "tokens");
+
+pub const ERC20_TOKEN_NAME: (&str, &str) = ("erc20Token", "");
+pub const ERC721_TOKEN_NAME: (&str, &str) = ("erc721Token", "");
+
pub const TOKEN_BALANCE_NAME: (&str, &str) = ("", "tokenBalances");
pub const TOKEN_TRANSFER_NAME: (&str, &str) = ("", "tokenTransfers");
-pub const ERC721_METADATA_NAME: (&str, &str) = ("erc721Metadata", "");
+// pub const ERC721_METADATA_NAME: (&str, &str) = ("erc721Metadata", "");
// misc
pub const ORDER_DIR_TYPE_NAME: &str = "OrderDirection";
diff --git a/crates/torii/graphql/src/mapping.rs b/crates/torii/graphql/src/mapping.rs
index adb8b37bc1..190cbf4b62 100644
--- a/crates/torii/graphql/src/mapping.rs
+++ b/crates/torii/graphql/src/mapping.rs
@@ -4,9 +4,7 @@ use async_graphql::Name;
use dojo_types::primitive::Primitive;
use lazy_static::lazy_static;
-use crate::constants::{
- CONTENT_TYPE_NAME, ERC721_METADATA_TYPE_NAME, SOCIAL_TYPE_NAME, TOKEN_TYPE_NAME,
-};
+use crate::constants::{CONTENT_TYPE_NAME, SOCIAL_TYPE_NAME, TOKEN_TYPE_NAME};
use crate::types::{GraphqlType, TypeData, TypeMapping};
lazy_static! {
@@ -148,38 +146,35 @@ lazy_static! {
]);
pub static ref TOKEN_BALANCE_TYPE_MAPPING: TypeMapping = IndexMap::from([
- (Name::new("balance"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("type"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("tokenMetadata"), TypeData::Simple(TypeRef::named_nn(TOKEN_TYPE_NAME))),
+ (Name::new("tokenMetadata"), TypeData::Nested((TypeRef::named_nn(TOKEN_TYPE_NAME), IndexMap::new()))),
]);
pub static ref TOKEN_TRANSFER_TYPE_MAPPING: TypeMapping = IndexMap::from([
(Name::new("from"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("to"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("amount"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("type"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("executedAt"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("tokenMetadata"), TypeData::Simple(TypeRef::named_nn(TOKEN_TYPE_NAME))),
+ (Name::new("tokenMetadata"), TypeData::Nested((TypeRef::named_nn(TOKEN_TYPE_NAME), IndexMap::new()))),
(Name::new("transactionHash"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
]);
- pub static ref TOKEN_TYPE_MAPPING: TypeMapping = IndexMap::from([
+ pub static ref ERC20_TOKEN_TYPE_MAPPING: TypeMapping = IndexMap::from([
(Name::new("name"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("symbol"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("decimals"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("contractAddress"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("decimals"), TypeData::Simple(TypeRef::named(TypeRef::STRING))),
- (
- Name::new("erc721"),
- TypeData::Nested((TypeRef::named(ERC721_METADATA_TYPE_NAME), IndexMap::new()))
- ),
+ (Name::new("amount"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
]);
- pub static ref ERC721_METADATA_TYPE_MAPPING: TypeMapping = IndexMap::from([
- (Name::new("tokenId"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ pub static ref ERC721_TOKEN_TYPE_MAPPING: TypeMapping = IndexMap::from([
(Name::new("name"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("description"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("attributes"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
- (Name::new("imagePath"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("symbol"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("tokenId"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("contractAddress"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
(Name::new("metadata"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("metadataName"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("metadataDescription"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("metadataAttributes"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
+ (Name::new("imagePath"), TypeData::Simple(TypeRef::named_nn(TypeRef::STRING))),
]);
+
}
diff --git a/crates/torii/graphql/src/object/erc/erc_token.rs b/crates/torii/graphql/src/object/erc/erc_token.rs
index d1a312b932..a3baeb10a6 100644
--- a/crates/torii/graphql/src/object/erc/erc_token.rs
+++ b/crates/torii/graphql/src/object/erc/erc_token.rs
@@ -1,40 +1,109 @@
-use crate::constants::{
- ERC721_METADATA_NAME, ERC721_METADATA_TYPE_NAME, TOKEN_NAME, TOKEN_TYPE_NAME,
-};
-use crate::mapping::{ERC721_METADATA_TYPE_MAPPING, TOKEN_TYPE_MAPPING};
+use async_graphql::dynamic::FieldValue;
+use async_graphql::{Name, Value};
+
+use crate::constants::{ERC20_TOKEN_NAME, ERC20_TYPE_NAME, ERC721_TOKEN_NAME, ERC721_TYPE_NAME};
+use crate::mapping::{ERC20_TOKEN_TYPE_MAPPING, ERC721_TOKEN_TYPE_MAPPING};
use crate::object::BasicObject;
-use crate::types::TypeMapping;
+use crate::types::{TypeMapping, ValueMapping};
#[derive(Debug)]
-pub struct ErcTokenObject;
+pub struct Erc20TokenObject;
-impl BasicObject for ErcTokenObject {
+impl BasicObject for Erc20TokenObject {
fn name(&self) -> (&str, &str) {
- TOKEN_NAME
+ ERC20_TOKEN_NAME
}
fn type_name(&self) -> &str {
- TOKEN_TYPE_NAME
+ ERC20_TYPE_NAME
}
fn type_mapping(&self) -> &TypeMapping {
- &TOKEN_TYPE_MAPPING
+ &ERC20_TOKEN_TYPE_MAPPING
}
}
#[derive(Debug)]
-pub struct Erc721MetadataObject;
+pub struct Erc721TokenObject;
-impl BasicObject for Erc721MetadataObject {
+impl BasicObject for Erc721TokenObject {
fn name(&self) -> (&str, &str) {
- ERC721_METADATA_NAME
+ ERC721_TOKEN_NAME
}
fn type_name(&self) -> &str {
- ERC721_METADATA_TYPE_NAME
+ ERC721_TYPE_NAME
}
fn type_mapping(&self) -> &TypeMapping {
- &ERC721_METADATA_TYPE_MAPPING
+ &ERC721_TOKEN_TYPE_MAPPING
+ }
+}
+
+#[derive(Debug, Clone)]
+pub enum ErcTokenType {
+ Erc20(Erc20Token),
+ Erc721(Erc721Token),
+}
+
+#[derive(Debug, Clone)]
+pub struct Erc20Token {
+ pub name: String,
+ pub symbol: String,
+ pub decimals: u8,
+ pub contract_address: String,
+ pub amount: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct Erc721Token {
+ pub name: String,
+ pub symbol: String,
+ pub token_id: String,
+ pub contract_address: String,
+ pub metadata: String,
+ pub metadata_name: Option<String>,
+ pub metadata_description: Option<String>,
+ pub metadata_attributes: Option<String>,
+ pub image_path: String,
+}
+
+impl ErcTokenType {
+ pub fn to_field_value<'a>(self) -> FieldValue<'a> {
+ match self {
+ ErcTokenType::Erc20(token) => FieldValue::with_type(
+ FieldValue::value(Value::Object(ValueMapping::from([
+ (Name::new("name"), Value::String(token.name)),
+ (Name::new("symbol"), Value::String(token.symbol)),
+ (Name::new("decimals"), Value::from(token.decimals)),
+ (Name::new("contractAddress"), Value::String(token.contract_address)),
+ (Name::new("amount"), Value::String(token.amount)),
+ ]))),
+ ERC20_TYPE_NAME.to_string(),
+ ),
+ ErcTokenType::Erc721(token) => FieldValue::with_type(
+ FieldValue::value(Value::Object(ValueMapping::from([
+ (Name::new("name"), Value::String(token.name)),
+ (Name::new("symbol"), Value::String(token.symbol)),
+ (Name::new("tokenId"), Value::String(token.token_id)),
+ (Name::new("contractAddress"), Value::String(token.contract_address)),
+ (Name::new("metadata"), Value::String(token.metadata)),
+ (
+ Name::new("metadataName"),
+ token.metadata_name.map(Value::String).unwrap_or(Value::Null),
+ ),
+ (
+ Name::new("metadataDescription"),
+ token.metadata_description.map(Value::String).unwrap_or(Value::Null),
+ ),
+ (
+ Name::new("metadataAttributes"),
+ token.metadata_attributes.map(Value::String).unwrap_or(Value::Null),
+ ),
+ (Name::new("imagePath"), Value::String(token.image_path)),
+ ]))),
+ ERC721_TYPE_NAME.to_string(),
+ ),
+ }
}
}
diff --git a/crates/torii/graphql/src/object/erc/mod.rs b/crates/torii/graphql/src/object/erc/mod.rs
index 72d0eb1fab..b5a4827582 100644
--- a/crates/torii/graphql/src/object/erc/mod.rs
+++ b/crates/torii/graphql/src/object/erc/mod.rs
@@ -1,3 +1,5 @@
+use async_graphql::Value;
+
use super::connection::cursor;
use crate::query::order::CursorDirection;
@@ -15,3 +17,16 @@ fn handle_cursor(
Err(_) => Err(sqlx::Error::Decode("Invalid cursor format".into())),
}
}
+
+#[derive(Debug, Clone)]
+pub struct ConnectionEdge<T> {
+ pub node: T,
+ pub cursor: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct Connection<T> {
+ pub total_count: i64,
+ pub edges: Vec<ConnectionEdge<T>>,
+ pub page_info: Value,
+}
diff --git a/crates/torii/graphql/src/object/erc/token_balance.rs b/crates/torii/graphql/src/object/erc/token_balance.rs
index 44818d9eee..7fdcca4fe5 100644
--- a/crates/torii/graphql/src/object/erc/token_balance.rs
+++ b/crates/torii/graphql/src/object/erc/token_balance.rs
@@ -1,6 +1,5 @@
use async_graphql::connection::PageInfo;
-use async_graphql::dynamic::{Field, FieldFuture, InputValue, TypeRef};
-use async_graphql::{Name, Value};
+use async_graphql::dynamic::{Field, FieldFuture, FieldValue, InputValue, TypeRef};
use convert_case::{Case, Casing};
use serde::Deserialize;
use sqlx::sqlite::SqliteRow;
@@ -10,18 +9,20 @@ use torii_core::constants::TOKEN_BALANCE_TABLE;
use torii_core::sql::utils::felt_to_sql_string;
use tracing::warn;
-use super::handle_cursor;
+use super::erc_token::{Erc20Token, ErcTokenType};
+use super::{handle_cursor, Connection, ConnectionEdge};
use crate::constants::{DEFAULT_LIMIT, ID_COLUMN, TOKEN_BALANCE_NAME, TOKEN_BALANCE_TYPE_NAME};
use crate::mapping::TOKEN_BALANCE_TYPE_MAPPING;
use crate::object::connection::page_info::PageInfoObject;
use crate::object::connection::{
connection_arguments, cursor, parse_connection_arguments, ConnectionArguments,
};
+use crate::object::erc::erc_token::Erc721Token;
use crate::object::{BasicObject, ResolvableObject};
use crate::query::data::count_rows;
use crate::query::filter::{Comparator, Filter, FilterValue};
use crate::query::order::{CursorDirection, Direction};
-use crate::types::{TypeMapping, ValueMapping};
+use crate::types::TypeMapping;
use crate::utils::extract;
#[derive(Debug)]
@@ -75,7 +76,7 @@ impl ResolvableObject for ErcBalanceObject {
let results = token_balances_connection_output(&data, total_count, page_info)?;
- Ok(Some(Value::Object(results)))
+ Ok(Some(results))
})
},
)
@@ -206,11 +207,11 @@ async fn fetch_token_balances(
}
}
-fn token_balances_connection_output(
+fn token_balances_connection_output<'a>(
data: &[SqliteRow],
total_count: i64,
page_info: PageInfo,
-) -> sqlx::Result<ValueMapping> {
+) -> sqlx::Result<FieldValue<'a>> {
let mut edges = Vec::new();
for row in data {
let row = BalanceQueryResultRaw::from_row(row)?;
@@ -218,19 +219,15 @@ fn token_balances_connection_output(
let balance_value = match row.contract_type.to_lowercase().as_str() {
"erc20" => {
- let token_metadata = Value::Object(ValueMapping::from([
- (Name::new("name"), Value::String(row.name)),
- (Name::new("symbol"), Value::String(row.symbol)),
- (Name::new("decimals"), Value::String(row.decimals.to_string())),
- (Name::new("contractAddress"), Value::String(row.contract_address.clone())),
- (Name::new("erc721"), Value::Null),
- ]));
-
- Value::Object(ValueMapping::from([
- (Name::new("balance"), Value::String(row.balance)),
- (Name::new("type"), Value::String(row.contract_type)),
- (Name::new("tokenMetadata"), token_metadata),
- ]))
+ let token_metadata = Erc20Token {
+ contract_address: row.contract_address,
+ name: row.name,
+ symbol: row.symbol,
+ decimals: row.decimals,
+ amount: row.balance,
+ };
+
+ ErcTokenType::Erc20(token_metadata)
}
"erc721" => {
// contract_address:token_id
@@ -239,47 +236,29 @@ fn token_balances_connection_output(
let metadata: serde_json::Value =
serde_json::from_str(&row.metadata).expect("metadata is always json");
- let erc721_name =
+ let metadata_name =
metadata.get("name").map(|v| v.to_string().trim_matches('"').to_string());
- let erc721_description = metadata
+ let metadata_description = metadata
.get("description")
.map(|v| v.to_string().trim_matches('"').to_string());
- let erc721_attributes =
+ let metadata_attributes =
metadata.get("attributes").map(|v| v.to_string().trim_matches('"').to_string());
let image_path = format!("{}/{}", token_id.join("/"), "image");
- let token_metadata = Value::Object(ValueMapping::from([
- (Name::new("contractAddress"), Value::String(row.contract_address.clone())),
- (Name::new("name"), Value::String(row.name)),
- (Name::new("symbol"), Value::String(row.symbol)),
- (Name::new("decimals"), Value::String(row.decimals.to_string())),
- (
- Name::new("erc721"),
- Value::Object(ValueMapping::from([
- (Name::new("imagePath"), Value::String(image_path)),
- (Name::new("tokenId"), Value::String(token_id[1].to_string())),
- (Name::new("metadata"), Value::String(row.metadata)),
- (
- Name::new("name"),
- erc721_name.map(Value::String).unwrap_or(Value::Null),
- ),
- (
- Name::new("description"),
- erc721_description.map(Value::String).unwrap_or(Value::Null),
- ),
- (
- Name::new("attributes"),
- erc721_attributes.map(Value::String).unwrap_or(Value::Null),
- ),
- ])),
- ),
- ]));
-
- Value::Object(ValueMapping::from([
- (Name::new("balance"), Value::String(row.balance)),
- (Name::new("type"), Value::String(row.contract_type)),
- (Name::new("tokenMetadata"), token_metadata),
- ]))
+
+ let token_metadata = Erc721Token {
+ name: row.name,
+ metadata: row.metadata,
+ contract_address: row.contract_address,
+ symbol: row.symbol,
+ token_id: token_id[1].to_string(),
+ metadata_name,
+ metadata_description,
+ metadata_attributes,
+ image_path,
+ };
+
+ ErcTokenType::Erc721(token_metadata)
}
_ => {
warn!("Unknown contract type: {}", row.contract_type);
@@ -287,17 +266,14 @@ fn token_balances_connection_output(
}
};
- edges.push(Value::Object(ValueMapping::from([
- (Name::new("node"), balance_value),
- (Name::new("cursor"), Value::String(cursor)),
- ])));
+ edges.push(ConnectionEdge { node: balance_value, cursor });
}
- Ok(ValueMapping::from([
- (Name::new("totalCount"), Value::from(total_count)),
- (Name::new("edges"), Value::List(edges)),
- (Name::new("pageInfo"), PageInfoObject::value(page_info)),
- ]))
+ Ok(FieldValue::owned_any(Connection {
+ total_count,
+ edges,
+ page_info: PageInfoObject::value(page_info),
+ }))
}
// TODO: This would be required when subscriptions are needed
diff --git a/crates/torii/graphql/src/object/erc/token_transfer.rs b/crates/torii/graphql/src/object/erc/token_transfer.rs
index 2dc8d8bd13..1441937c2e 100644
--- a/crates/torii/graphql/src/object/erc/token_transfer.rs
+++ b/crates/torii/graphql/src/object/erc/token_transfer.rs
@@ -1,6 +1,5 @@
use async_graphql::connection::PageInfo;
-use async_graphql::dynamic::{Field, FieldFuture, InputValue, TypeRef};
-use async_graphql::{Name, Value};
+use async_graphql::dynamic::{Field, FieldFuture, FieldValue, InputValue, TypeRef};
use convert_case::{Case, Casing};
use serde::Deserialize;
use sqlx::sqlite::SqliteRow;
@@ -11,16 +10,18 @@ use torii_core::engine::get_transaction_hash_from_event_id;
use torii_core::sql::utils::felt_to_sql_string;
use tracing::warn;
-use super::handle_cursor;
+use super::erc_token::{Erc20Token, ErcTokenType};
+use super::{handle_cursor, Connection, ConnectionEdge};
use crate::constants::{DEFAULT_LIMIT, ID_COLUMN, TOKEN_TRANSFER_NAME, TOKEN_TRANSFER_TYPE_NAME};
use crate::mapping::TOKEN_TRANSFER_TYPE_MAPPING;
use crate::object::connection::page_info::PageInfoObject;
use crate::object::connection::{
connection_arguments, cursor, parse_connection_arguments, ConnectionArguments,
};
+use crate::object::erc::erc_token::Erc721Token;
use crate::object::{BasicObject, ResolvableObject};
use crate::query::order::{CursorDirection, Direction};
-use crate::types::{TypeMapping, ValueMapping};
+use crate::types::TypeMapping;
use crate::utils::extract;
#[derive(Debug)]
@@ -74,7 +75,7 @@ impl ResolvableObject for ErcTransferObject {
fetch_token_transfers(&mut conn, address, &connection, total_count).await?;
let results = token_transfers_connection_output(&data, total_count, page_info)?;
- Ok(Some(Value::Object(results)))
+ Ok(Some(results))
})
},
)
@@ -227,11 +228,11 @@ JOIN
}
}
-fn token_transfers_connection_output(
+fn token_transfers_connection_output<'a>(
data: &[SqliteRow],
total_count: i64,
page_info: PageInfo,
-) -> sqlx::Result<ValueMapping> {
+) -> sqlx::Result<FieldValue<'a>> {
let mut edges = Vec::new();
for row in data {
@@ -239,80 +240,60 @@ fn token_transfers_connection_output(
let transaction_hash = get_transaction_hash_from_event_id(&row.id);
let cursor = cursor::encode(&row.id, &row.id);
- let transfer_value = match row.contract_type.to_lowercase().as_str() {
+ let transfer_node = match row.contract_type.to_lowercase().as_str() {
"erc20" => {
- let token_metadata = Value::Object(ValueMapping::from([
- (Name::new("name"), Value::String(row.name)),
- (Name::new("symbol"), Value::String(row.symbol)),
- // for erc20 there is no token_id
- (Name::new("tokenId"), Value::Null),
- (Name::new("decimals"), Value::String(row.decimals.to_string())),
- (Name::new("contractAddress"), Value::String(row.contract_address.clone())),
- (Name::new("erc721"), Value::Null),
- ]));
-
- Value::Object(ValueMapping::from([
- (Name::new("from"), Value::String(row.from_address)),
- (Name::new("to"), Value::String(row.to_address)),
- (Name::new("amount"), Value::String(row.amount)),
- (Name::new("type"), Value::String(row.contract_type)),
- (Name::new("executedAt"), Value::String(row.executed_at)),
- (Name::new("tokenMetadata"), token_metadata),
- (Name::new("transactionHash"), Value::String(transaction_hash)),
- ]))
+ let token_metadata = ErcTokenType::Erc20(Erc20Token {
+ contract_address: row.contract_address,
+ name: row.name,
+ symbol: row.symbol,
+ decimals: row.decimals,
+ amount: row.amount,
+ });
+
+ TokenTransferNode {
+ from: row.from_address,
+ to: row.to_address,
+ executed_at: row.executed_at,
+ token_metadata,
+ transaction_hash,
+ }
}
"erc721" => {
// contract_address:token_id
let token_id = row.token_id.split(':').collect::<Vec<&str>>();
assert!(token_id.len() == 2);
- let image_path = format!("{}/{}", token_id.join("/"), "image");
let metadata: serde_json::Value =
serde_json::from_str(&row.metadata).expect("metadata is always json");
- let erc721_name =
+ let metadata_name =
metadata.get("name").map(|v| v.to_string().trim_matches('"').to_string());
- let erc721_description = metadata
+ let metadata_description = metadata
.get("description")
.map(|v| v.to_string().trim_matches('"').to_string());
- let erc721_attributes =
+ let metadata_attributes =
metadata.get("attributes").map(|v| v.to_string().trim_matches('"').to_string());
- let token_metadata = Value::Object(ValueMapping::from([
- (Name::new("name"), Value::String(row.name)),
- (Name::new("symbol"), Value::String(row.symbol)),
- (Name::new("decimals"), Value::String(row.decimals.to_string())),
- (Name::new("contractAddress"), Value::String(row.contract_address.clone())),
- (
- Name::new("erc721"),
- Value::Object(ValueMapping::from([
- (Name::new("imagePath"), Value::String(image_path)),
- (Name::new("tokenId"), Value::String(token_id[1].to_string())),
- (Name::new("metadata"), Value::String(row.metadata)),
- (
- Name::new("name"),
- erc721_name.map(Value::String).unwrap_or(Value::Null),
- ),
- (
- Name::new("description"),
- erc721_description.map(Value::String).unwrap_or(Value::Null),
- ),
- (
- Name::new("attributes"),
- erc721_attributes.map(Value::String).unwrap_or(Value::Null),
- ),
- ])),
- ),
- ]));
-
- Value::Object(ValueMapping::from([
- (Name::new("from"), Value::String(row.from_address)),
- (Name::new("to"), Value::String(row.to_address)),
- (Name::new("amount"), Value::String(row.amount)),
- (Name::new("type"), Value::String(row.contract_type)),
- (Name::new("executedAt"), Value::String(row.executed_at)),
- (Name::new("tokenMetadata"), token_metadata),
- (Name::new("transactionHash"), Value::String(transaction_hash)),
- ]))
+ let image_path = format!("{}/{}", token_id.join("/"), "image");
+
+ let token_metadata = ErcTokenType::Erc721(Erc721Token {
+ name: row.name,
+ metadata: row.metadata,
+ contract_address: row.contract_address,
+ symbol: row.symbol,
+ token_id: token_id[1].to_string(),
+ metadata_name,
+ metadata_description,
+ metadata_attributes,
+ image_path,
+ });
+
+ TokenTransferNode {
+ from: row.from_address,
+ to: row.to_address,
+ executed_at: row.executed_at,
+ token_metadata,
+ transaction_hash,
+ }
}
_ => {
warn!("Unknown contract type: {}", row.contract_type);
@@ -320,17 +301,14 @@ fn token_transfers_connection_output(
}
};
- edges.push(Value::Object(ValueMapping::from([
- (Name::new("node"), transfer_value),
- (Name::new("cursor"), Value::String(cursor)),
- ])));
+ edges.push(ConnectionEdge { node: transfer_node, cursor });
}
- Ok(ValueMapping::from([
- (Name::new("totalCount"), Value::from(total_count)),
- (Name::new("edges"), Value::List(edges)),
- (Name::new("pageInfo"), PageInfoObject::value(page_info)),
- ]))
+ Ok(FieldValue::owned_any(Connection {
+ total_count,
+ edges,
+ page_info: PageInfoObject::value(page_info),
+ }))
}
// TODO: This would be required when subscriptions are needed
@@ -357,3 +335,12 @@ struct TransferQueryResultRaw {
pub contract_type: String,
pub metadata: String,
}
+
+#[derive(Debug, Clone)]
+pub struct TokenTransferNode {
+ pub from: String,
+ pub to: String,
+ pub executed_at: String,
+ pub token_metadata: ErcTokenType,
+ pub transaction_hash: String,
+}
diff --git a/crates/torii/graphql/src/object/mod.rs b/crates/torii/graphql/src/object/mod.rs
index 8997cdabe3..36c1f9b754 100644
--- a/crates/torii/graphql/src/object/mod.rs
+++ b/crates/torii/graphql/src/object/mod.rs
@@ -10,10 +10,14 @@ pub mod model_data;
pub mod transaction;
use async_graphql::dynamic::{
- Enum, Field, FieldFuture, InputObject, InputValue, Object, SubscriptionField, TypeRef,
+ Enum, Field, FieldFuture, FieldValue, InputObject, InputValue, Object, SubscriptionField,
+ TypeRef,
};
use async_graphql::Value;
use convert_case::{Case, Casing};
+use erc::erc_token::ErcTokenType;
+use erc::token_transfer::TokenTransferNode;
+use erc::{Connection, ConnectionEdge};
use sqlx::{Pool, Sqlite};
use self::connection::edge::EdgeObject;
@@ -57,12 +61,133 @@ pub trait BasicObject: Send + Sync {
let field_name = field_name.clone();
FieldFuture::new(async move {
- match ctx.parent_value.try_to_value()? {
- Value::Object(values) => {
- Ok(Some(values.get(&field_name).unwrap().clone())) // safe unwrap
+ match ctx.parent_value.try_to_value() {
+ Ok(Value::Object(values)) => {
+ // safe unwrap
+ return Ok(Some(FieldValue::value(
+ values.get(&field_name).unwrap().clone(),
+ )));
+ }
+ // if the parent is `Value` then it must be a Object
+ Ok(_) => return Err("incorrect value, requires Value::Object".into()),
+ _ => {}
+ };
+
+ // if its not we try to downcast to known types which is a special case for
+ // tokenBalances and tokenTransfers queries
+
+ if let Ok(values) =
+ ctx.parent_value.try_downcast_ref::<Connection<ErcTokenType>>()
+ {
+ match field_name.as_str() {
+ "edges" => {
+ return Ok(Some(FieldValue::list(
+ values
+ .edges
+ .iter()
+ .map(|v| FieldValue::owned_any(v.clone()))
+ .collect::<Vec<FieldValue<'_>>>(),
+ )));
+ }
+ "pageInfo" => {
+ return Ok(Some(FieldValue::value(values.page_info.clone())));
+ }
+ "totalCount" => {
+ return Ok(Some(FieldValue::value(Value::from(
+ values.total_count,
+ ))));
+ }
+ _ => return Err("incorrect value, requires Value::Object".into()),
+ }
+ }
+
+ if let Ok(values) =
+ ctx.parent_value.try_downcast_ref::<ConnectionEdge<ErcTokenType>>()
+ {
+ match field_name.as_str() {
+ "node" => return Ok(Some(FieldValue::owned_any(values.node.clone()))),
+ "cursor" => {
+ return Ok(Some(FieldValue::value(Value::String(
+ values.cursor.clone(),
+ ))));
+ }
+ _ => return Err("incorrect value, requires Value::Object".into()),
+ }
+ }
+
+ if let Ok(values) =
+ ctx.parent_value.try_downcast_ref::<Connection<TokenTransferNode>>()
+ {
+ match field_name.as_str() {
+ "edges" => {
+ return Ok(Some(FieldValue::list(
+ values
+ .edges
+ .iter()
+ .map(|v| FieldValue::owned_any(v.clone()))
+ .collect::<Vec<FieldValue<'_>>>(),
+ )));
+ }
+ "pageInfo" => {
+ return Ok(Some(FieldValue::value(values.page_info.clone())));
+ }
+ "totalCount" => {
+ return Ok(Some(FieldValue::value(Value::from(
+ values.total_count,
+ ))));
+ }
+ _ => return Err("incorrect value, requires Value::Object".into()),
}
- _ => Err("incorrect value, requires Value::Object".into()),
}
+
+ if let Ok(values) =
+ ctx.parent_value.try_downcast_ref::<ConnectionEdge<TokenTransferNode>>()
+ {
+ match field_name.as_str() {
+ "node" => return Ok(Some(FieldValue::owned_any(values.node.clone()))),
+ "cursor" => {
+ return Ok(Some(FieldValue::value(Value::String(
+ values.cursor.clone(),
+ ))));
+ }
+ _ => return Err("incorrect value, requires Value::Object".into()),
+ }
+ }
+
+ if let Ok(values) = ctx.parent_value.try_downcast_ref::<TokenTransferNode>() {
+ match field_name.as_str() {
+ "from" => {
+ return Ok(Some(FieldValue::value(Value::String(
+ values.from.clone(),
+ ))));
+ }
+ "to" => {
+ return Ok(Some(FieldValue::value(Value::String(
+ values.to.clone(),
+ ))));
+ }
+ "executedAt" => {
+ return Ok(Some(FieldValue::value(Value::String(
+ values.executed_at.clone(),
+ ))));
+ }
+ "tokenMetadata" => {
+ return Ok(Some(values.clone().token_metadata.to_field_value()));
+ }
+ "transactionHash" => {
+ return Ok(Some(FieldValue::value(Value::String(
+ values.transaction_hash.clone(),
+ ))));
+ }
+ _ => return Err("incorrect value, requires Value::Object".into()),
+ }
+ }
+
+ if let Ok(values) = ctx.parent_value.try_downcast_ref::<ErcTokenType>() {
+ return Ok(Some(values.clone().to_field_value()));
+ }
+
+ Err("unexpected parent value".into())
})
});
diff --git a/crates/torii/graphql/src/schema.rs b/crates/torii/graphql/src/schema.rs
index 7d9b90b883..79ec29e15d 100644
--- a/crates/torii/graphql/src/schema.rs
+++ b/crates/torii/graphql/src/schema.rs
@@ -9,8 +9,10 @@ use super::object::event::EventObject;
use super::object::model_data::ModelDataObject;
use super::types::ScalarType;
use super::utils;
-use crate::constants::{QUERY_TYPE_NAME, SUBSCRIPTION_TYPE_NAME};
-use crate::object::erc::erc_token::{Erc721MetadataObject, ErcTokenObject};
+use crate::constants::{
+ ERC20_TYPE_NAME, ERC721_TYPE_NAME, QUERY_TYPE_NAME, SUBSCRIPTION_TYPE_NAME, TOKEN_TYPE_NAME,
+};
+use crate::object::erc::erc_token::{Erc20TokenObject, Erc721TokenObject};
use crate::object::erc::token_balance::ErcBalanceObject;
use crate::object::erc::token_transfer::ErcTransferObject;
use crate::object::event_message::EventMessageObject;
@@ -121,14 +123,20 @@ async fn build_objects(pool: &SqlitePool) -> Result<(Vec<ObjectVariant>, Vec<Uni
ObjectVariant::Basic(Box::new(SocialObject)),
ObjectVariant::Basic(Box::new(ContentObject)),
ObjectVariant::Basic(Box::new(PageInfoObject)),
- ObjectVariant::Basic(Box::new(ErcTokenObject)),
- ObjectVariant::Basic(Box::new(Erc721MetadataObject)),
+ ObjectVariant::Basic(Box::new(Erc721TokenObject)),
+ ObjectVariant::Basic(Box::new(Erc20TokenObject)),
];
// model union object
let mut unions: Vec<Union> = Vec::new();
let mut model_union = Union::new("ModelUnion");
+ // erc_token union object
+ let erc_token_union =
+ Union::new(TOKEN_TYPE_NAME).possible_type(ERC20_TYPE_NAME).possible_type(ERC721_TYPE_NAME);
+
+ unions.push(erc_token_union);
+
// model data objects
for model in models {
let type_mapping = type_mapping_query(&mut conn, &model.id).await?;
From ae4eabb8ee93b6b57024bbd04915b2e2b939cfb3 Mon Sep 17 00:00:00 2001
From: lambda-0x <0xlambda@protonmail.com>
Date: Mon, 4 Nov 2024 02:30:41 +0530
Subject: [PATCH 6/9] fix(torii/core): properly update transaction hash while
processing pending block
commit-id:fd50508d
---
crates/torii/core/src/engine.rs | 1 +
1 file changed, 1 insertion(+)
diff --git a/crates/torii/core/src/engine.rs b/crates/torii/core/src/engine.rs
index 36d7d788fb..6ea55ce344 100644
--- a/crates/torii/core/src/engine.rs
+++ b/crates/torii/core/src/engine.rs
@@ -737,6 +737,7 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
for contract in unique_contracts {
let entry = cursor_map.entry(contract).or_insert((*transaction_hash, 0));
+ entry.0 = *transaction_hash;
entry.1 += 1;
}
From d4daef35a5013f2daffd120577aeef6d7e38e6cb Mon Sep 17 00:00:00 2001
From: lambda-0x <0xlambda@protonmail.com>
Date: Mon, 4 Nov 2024 02:37:24 +0530
Subject: [PATCH 7/9] fix(torii/core): rollback transaction when engine retries
commit-id:16f58fa9
---
crates/torii/core/src/engine.rs | 2 ++
crates/torii/core/src/executor/mod.rs | 39 +++++++++++++++++++++++++++
crates/torii/core/src/sql/mod.rs | 6 +++++
3 files changed, 47 insertions(+)
diff --git a/crates/torii/core/src/engine.rs b/crates/torii/core/src/engine.rs
index 6ea55ce344..0d91e95ed3 100644
--- a/crates/torii/core/src/engine.rs
+++ b/crates/torii/core/src/engine.rs
@@ -284,6 +284,8 @@ impl<P: Provider + Send + Sync + std::fmt::Debug + 'static> Engine<P> {
Err(e) => {
error!(target: LOG_TARGET, error = %e, "Processing fetched data.");
erroring_out = true;
+ // incase of error rollback the transaction
+ self.db.rollback().await?;
sleep(backoff_delay).await;
if backoff_delay < max_backoff_delay {
backoff_delay *= 2;
diff --git a/crates/torii/core/src/executor/mod.rs b/crates/torii/core/src/executor/mod.rs
index c823aa1255..18c76f6d64 100644
--- a/crates/torii/core/src/executor/mod.rs
+++ b/crates/torii/core/src/executor/mod.rs
@@ -116,6 +116,8 @@ pub enum QueryType {
// similar to execute but doesn't create a new transaction
Flush,
Execute,
+ // rollback's the current transaction and starts a new one
+ Rollback,
Other,
}
@@ -208,6 +210,19 @@ impl QueryMessage {
rx,
)
}
+
+ pub fn rollback_recv() -> (Self, oneshot::Receiver<Result<()>>) {
+ let (tx, rx) = oneshot::channel();
+ (
+ Self {
+ statement: "".to_string(),
+ arguments: vec![],
+ query_type: QueryType::Rollback,
+ tx: Some(tx),
+ },
+ rx,
+ )
+ }
}
impl<'c, P: Provider + Sync + Send + 'static> Executor<'c, P> {
@@ -733,6 +748,20 @@ impl<'c, P: Provider + Sync + Send + 'static> Executor<'c, P> {
// defer executing these queries since they depend on TokenRegister queries
self.deferred_query_messages.push(query_message);
}
+ QueryType::Rollback => {
+ debug!(target: LOG_TARGET, "Rolling back the transaction.");
+ // rollback's the current transaction and starts a new one
+ let res = self.rollback().await;
+ debug!(target: LOG_TARGET, "Rolled back the transaction.");
+
+ if let Some(sender) = query_message.tx {
+ sender
+ .send(res)
+ .map_err(|_| anyhow::anyhow!("Failed to send rollback result"))?;
+ } else {
+ res?;
+ }
+ }
QueryType::Other => {
query.execute(&mut **tx).await.with_context(|| {
format!(
@@ -785,6 +814,16 @@ impl<'c, P: Provider + Sync + Send + 'static> Executor<'c, P> {
Ok(())
}
+
+ async fn rollback(&mut self) -> Result<()> {
+ let transaction = mem::replace(&mut self.transaction, self.pool.begin().await?);
+ transaction.rollback().await?;
+
+ // NOTE: clear doesn't reset the capacity
+ self.publish_queue.clear();
+ self.deferred_query_messages.clear();
+ Ok(())
+ }
}
fn send_broker_message(message: BrokerMessage) {
diff --git a/crates/torii/core/src/sql/mod.rs b/crates/torii/core/src/sql/mod.rs
index 11eedf6a3c..9c61405d99 100644
--- a/crates/torii/core/src/sql/mod.rs
+++ b/crates/torii/core/src/sql/mod.rs
@@ -1311,4 +1311,10 @@ impl Sql {
self.executor.send(flush)?;
recv.await?
}
+
+ pub async fn rollback(&self) -> Result<()> {
+ let (rollback, recv) = QueryMessage::rollback_recv();
+ self.executor.send(rollback)?;
+ recv.await?
+ }
}
From 8a7a9a23806dd1c0a2347e6d1a8b5e2c83549c62 Mon Sep 17 00:00:00 2001
From: lambda-0x <0xlambda@protonmail.com>
Date: Thu, 14 Nov 2024 06:10:33 +0700
Subject: [PATCH 8/9] fix: git rebase
commit-id:ddf37cb7
---
Cargo.lock | 28 ++++
bin/torii/src/main.rs | 128 ++----------------
crates/torii/cli/Cargo.toml | 5 +-
crates/torii/cli/src/args.rs | 5 +
.../graphql/src/tests/subscription_test.rs | 1 -
5 files changed, 48 insertions(+), 119 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 84744fddd2..e4f92c298e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -10099,6 +10099,34 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8"
+[[package]]
+name = "notify"
+version = "7.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009"
+dependencies = [
+ "bitflags 2.6.0",
+ "filetime",
+ "fsevent-sys",
+ "inotify",
+ "kqueue",
+ "libc",
+ "log",
+ "mio",
+ "notify-types",
+ "walkdir",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "notify-types"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7393c226621f817964ffb3dc5704f9509e107a8b024b489cc2c1b217378785df"
+dependencies = [
+ "instant",
+]
+
[[package]]
name = "ntapi"
version = "0.4.1"
diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs
index c605d58079..ff15a88644 100644
--- a/bin/torii/src/main.rs
+++ b/bin/torii/src/main.rs
@@ -16,10 +16,8 @@ use std::str::FromStr;
use std::sync::Arc;
use std::time::Duration;
-use anyhow::Context;
use camino::Utf8PathBuf;
use clap::Parser;
-use clap::{ArgAction, Parser};
use dojo_metrics::exporters::prometheus::PrometheusRecorder;
use dojo_world::contracts::world::WorldContractReader;
use sqlx::sqlite::{
@@ -48,111 +46,6 @@ use url::{form_urlencoded, Url};
pub(crate) const LOG_TARGET: &str = "torii::cli";
-/// Dojo World Indexer
-#[derive(Parser, Debug)]
-#[command(name = "torii", author, version, about, long_about = None)]
-struct Args {
- /// The world to index
- #[arg(short, long = "world", env = "DOJO_WORLD_ADDRESS")]
- world_address: Option<Felt>,
-
- /// The sequencer rpc endpoint to index.
- #[arg(long, value_name = "URL", default_value = ":5050", value_parser = parse_url)]
- rpc: Url,
-
- /// Database filepath (ex: indexer.db). If specified file doesn't exist, it will be
- /// created. Defaults to in-memory database
- #[arg(short, long, default_value = "")]
- database: String,
-
- /// Address to serve api endpoints at.
- #[arg(long, value_name = "SOCKET", default_value = "0.0.0.0:8080", value_parser = parse_socket_address)]
- addr: SocketAddr,
-
- /// Port to serve Libp2p TCP & UDP Quic transports
- #[arg(long, value_name = "PORT", default_value = "9090")]
- relay_port: u16,
-
- /// Port to serve Libp2p WebRTC transport
- #[arg(long, value_name = "PORT", default_value = "9091")]
- relay_webrtc_port: u16,
-
- /// Port to serve Libp2p WebRTC transport
- #[arg(long, value_name = "PORT", default_value = "9092")]
- relay_websocket_port: u16,
-
- /// Path to a local identity key file. If not specified, a new identity will be generated
- #[arg(long, value_name = "PATH")]
- relay_local_key_path: Option<String>,
-
- /// Path to a local certificate file. If not specified, a new certificate will be generated
- /// for WebRTC connections
- #[arg(long, value_name = "PATH")]
- relay_cert_path: Option<String>,
-
- /// Specify allowed origins for api endpoints (comma-separated list of allowed origins, or "*"
- /// for all)
- #[arg(long)]
- #[arg(value_delimiter = ',')]
- allowed_origins: Option<Vec<String>>,
-
- /// The external url of the server, used for configuring the GraphQL Playground in a hosted
- /// environment
- #[arg(long, value_parser = parse_url)]
- external_url: Option<Url>,
-
- /// Enable Prometheus metrics.
- ///
- /// The metrics will be served at the given interface and port.
- #[arg(long, value_name = "SOCKET", value_parser = parse_socket_address, help_heading = "Metrics")]
- metrics: Option<SocketAddr>,
-
- /// Open World Explorer on the browser.
- #[arg(long)]
- explorer: bool,
-
- /// Chunk size of the events page when indexing using events
- #[arg(long, default_value = "1024")]
- events_chunk_size: u64,
-
- /// Number of blocks to process before commiting to DB
- #[arg(long, default_value = "10240")]
- blocks_chunk_size: u64,
-
- /// Enable indexing pending blocks
- #[arg(long, action = ArgAction::Set, default_value_t = true)]
- index_pending: bool,
-
- /// Polling interval in ms
- #[arg(long, default_value = "500")]
- polling_interval: u64,
-
- /// Max concurrent tasks
- #[arg(long, default_value = "100")]
- max_concurrent_tasks: usize,
-
- /// Whether or not to index world transactions
- #[arg(long, action = ArgAction::Set, default_value_t = false)]
- index_transactions: bool,
-
- /// Whether or not to index raw events
- #[arg(long, action = ArgAction::Set, default_value_t = true)]
- index_raw_events: bool,
-
- /// ERC contract addresses to index
- #[arg(long, value_parser = parse_erc_contracts)]
- #[arg(conflicts_with = "config")]
- contracts: Option<std::vec::Vec<Contract>>,
-
- /// Configuration file
- #[arg(long)]
- config: Option<PathBuf>,
-
- /// Path to a directory to store ERC artifacts
- #[arg(long)]
- artifacts_path: Option<Utf8PathBuf>,
-}
-
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let mut args = ToriiArgs::parse().with_config_file()?;
@@ -217,19 +110,11 @@ async fn main() -> anyhow::Result<()> {
// Get world address
let world = WorldContractReader::new(world_address, provider.clone());
- // let (mut executor, sender) = Executor::new(pool.clone(), shutdown_tx.clone()).await?;
- let contracts = args
- .indexing
- .contracts
- .iter()
- .map(|contract| (contract.address, contract.r#type))
- .collect();
-
let (mut executor, sender) = Executor::new(
pool.clone(),
shutdown_tx.clone(),
provider.clone(),
- args.max_concurrent_tasks,
+ args.indexing.max_concurrent_tasks,
)
.await?;
let executor_handle = tokio::spawn(async move { executor.run().await });
@@ -287,6 +172,16 @@ async fn main() -> anyhow::Result<()> {
)
.await?;
+ let temp_dir = TempDir::new()?;
+ let artifacts_path =
+ args.artifacts_path.unwrap_or_else(|| Utf8PathBuf::from(temp_dir.path().to_str().unwrap()));
+
+ tokio::fs::create_dir_all(&artifacts_path).await?;
+ let absolute_path = artifacts_path.canonicalize_utf8()?;
+
+ let (artifacts_addr, artifacts_server) =
+ torii_server::artifacts::new(shutdown_tx.subscribe(), &absolute_path, pool.clone()).await?;
+
let mut libp2p_relay_server = torii_relay::server::Relay::new(
db,
provider.clone(),
@@ -299,6 +194,7 @@ async fn main() -> anyhow::Result<()> {
.expect("Failed to start libp2p relay server");
let addr = SocketAddr::new(args.server.http_addr, args.server.http_port);
+
let proxy_server = Arc::new(Proxy::new(
addr,
args.server.http_cors_origins.filter(|cors_origins| !cors_origins.is_empty()),
diff --git a/crates/torii/cli/Cargo.toml b/crates/torii/cli/Cargo.toml
index 0f8b708b33..e4ed7ab09e 100644
--- a/crates/torii/cli/Cargo.toml
+++ b/crates/torii/cli/Cargo.toml
@@ -7,12 +7,13 @@ version.workspace = true
[dependencies]
anyhow.workspace = true
+camino.workspace = true
clap.workspace = true
dojo-utils.workspace = true
serde.workspace = true
starknet.workspace = true
-torii-core.workspace = true
toml.workspace = true
+torii-core.workspace = true
url.workspace = true
[dev-dependencies]
@@ -21,4 +22,4 @@ camino.workspace = true
[features]
default = [ "server" ]
-server = [ ]
+server = [ ]
diff --git a/crates/torii/cli/src/args.rs b/crates/torii/cli/src/args.rs
index 64dfb58b0c..8749ab5e2d 100644
--- a/crates/torii/cli/src/args.rs
+++ b/crates/torii/cli/src/args.rs
@@ -1,6 +1,7 @@
use std::path::PathBuf;
use anyhow::Result;
+use camino::Utf8PathBuf;
use clap::Parser;
use dojo_utils::parse::parse_url;
use serde::{Deserialize, Serialize};
@@ -47,6 +48,10 @@ pub struct ToriiArgs {
#[arg(long, help = "Configuration file to setup Torii.")]
pub config: Option<PathBuf>,
+ /// Path to a directory to store ERC artifacts
+ #[arg(long)]
+ pub artifacts_path: Option<Utf8PathBuf>,
+
#[command(flatten)]
pub indexing: IndexingOptions,
diff --git a/crates/torii/graphql/src/tests/subscription_test.rs b/crates/torii/graphql/src/tests/subscription_test.rs
index 39146feddf..bc054f059c 100644
--- a/crates/torii/graphql/src/tests/subscription_test.rs
+++ b/crates/torii/graphql/src/tests/subscription_test.rs
@@ -20,7 +20,6 @@ mod tests {
use torii_core::sql::cache::ModelCache;
use torii_core::sql::utils::felts_to_sql_string;
use torii_core::sql::Sql;
- use torii_core::types::ContractType;
use torii_core::types::{Contract, ContractType};
use url::Url;
From 705de8837113338088c320cf5872fa242c7adb66 Mon Sep 17 00:00:00 2001
From: lambda-0x <0xlambda@protonmail.com>
Date: Thu, 14 Nov 2024 15:40:55 +0530
Subject: [PATCH 9/9] fix(torii/graphql): use borrowed_any instead of owned_any
commit-id:eaaead2c
---
crates/torii/graphql/src/object/mod.rs | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/crates/torii/graphql/src/object/mod.rs b/crates/torii/graphql/src/object/mod.rs
index 36c1f9b754..b4201e0989 100644
--- a/crates/torii/graphql/src/object/mod.rs
+++ b/crates/torii/graphql/src/object/mod.rs
@@ -85,7 +85,7 @@ pub trait BasicObject: Send + Sync {
values
.edges
.iter()
- .map(|v| FieldValue::owned_any(v.clone()))
+ .map(FieldValue::borrowed_any)
.collect::<Vec<FieldValue<'_>>>(),
)));
}
@@ -105,7 +105,7 @@ pub trait BasicObject: Send + Sync {
ctx.parent_value.try_downcast_ref::<ConnectionEdge<ErcTokenType>>()
{
match field_name.as_str() {
- "node" => return Ok(Some(FieldValue::owned_any(values.node.clone()))),
+ "node" => return Ok(Some(FieldValue::borrowed_any(&values.node))),
"cursor" => {
return Ok(Some(FieldValue::value(Value::String(
values.cursor.clone(),
@@ -124,7 +124,7 @@ pub trait BasicObject: Send + Sync {
values
.edges
.iter()
- .map(|v| FieldValue::owned_any(v.clone()))
+ .map(FieldValue::borrowed_any)
.collect::<Vec<FieldValue<'_>>>(),
)));
}
@@ -144,7 +144,7 @@ pub trait BasicObject: Send + Sync {
ctx.parent_value.try_downcast_ref::<ConnectionEdge<TokenTransferNode>>()
{
match field_name.as_str() {
- "node" => return Ok(Some(FieldValue::owned_any(values.node.clone()))),
+ "node" => return Ok(Some(FieldValue::borrowed_any(&values.node))),
"cursor" => {
return Ok(Some(FieldValue::value(Value::String(
values.cursor.clone(),