feat: store msg types inside message_type table and add messages_by_type function

CHANGELOG.md

@@ -1,4 +1,5 @@
 ## Unreleased
+- ([\#105](https://github.com/forbole/juno/pull/105)) Stored messages types inside new `message_type` table, added new `messages_by_type` function and added v5 migrator that allows to parse all msg types already stored in db
 
 ### Changes
 - ([\#100](https://github.com/forbole/juno/pull/100)) improve account relationship mapping

cmd/migrate/cmd.go

@@ -9,13 +9,15 @@ import (
 	"github.com/spf13/cobra"
 
 	v4 "github.com/forbole/juno/v5/cmd/migrate/v4"
+	v5 "github.com/forbole/juno/v5/cmd/migrate/v5"
 )
 
 type Migrator func(parseCfg *parsecmdtypes.Config) error
 
 var (
 	migrations = map[string]Migrator{
 		"v4": v4.RunMigration,
+		"v5": v5.RunMigration,
 	}
 )
 

cmd/migrate/v4/migrate.go

@@ -8,6 +8,7 @@ import (
 
 	"gopkg.in/yaml.v3"
 
+	utils "github.com/forbole/juno/v5/cmd/migrate/utils"
 	v3 "github.com/forbole/juno/v5/cmd/migrate/v3"
 	databaseconfig "github.com/forbole/juno/v5/database/config"
 	"github.com/forbole/juno/v5/types/config"
@@ -40,10 +41,10 @@ func RunMigration(parseConfig *parsecmdtypes.Config) error {
 	return nil
 }
 
-func migrateConfig() (Config, error) {
+func migrateConfig() (utils.Config, error) {
 	cfg, err := v3.GetConfig()
 	if err != nil {
-		return Config{}, fmt.Errorf("error while reading v3 config: %s", err)
+		return utils.Config{}, fmt.Errorf("error while reading v3 config: %s", err)
 	}
 
 	sslMode := cfg.Database.SSLMode
@@ -56,7 +57,7 @@ func migrateConfig() (Config, error) {
 		schema = "public"
 	}
 
-	return Config{
+	return utils.Config{
 		Node:  cfg.Node,
 		Chain: cfg.Chain,
 		Database: databaseconfig.Config{

cmd/migrate/v5/migrate.go

@@ -0,0 +1,44 @@
+package v5
+
+import (
+	"fmt"
+
+	utils "github.com/forbole/juno/v5/cmd/migrate/utils"
+
+	parse "github.com/forbole/juno/v5/cmd/parse/types"
+	"github.com/forbole/juno/v5/database"
+	v5db "github.com/forbole/juno/v5/database/legacy/v5"
+	"github.com/forbole/juno/v5/database/postgresql"
+)
+
+// RunMigration runs the migrations to v5
+func RunMigration(parseConfig *parse.Config) error {
+	cfg, err := GetConfig()
+	if err != nil {
+		return fmt.Errorf("error while reading config: %s", err)
+	}
+
+	// Migrate the database
+	err = migrateDb(cfg, parseConfig)
+	if err != nil {
+		return fmt.Errorf("error while migrating database: %s", err)
+	}
+
+	return nil
+}
+
+func migrateDb(cfg utils.Config, parseConfig *parse.Config) error {
+	// Build the codec
+	encodingConfig := parseConfig.GetEncodingConfigBuilder()()
+
+	// Get the db
+	databaseCtx := database.NewContext(cfg.Database, encodingConfig, parseConfig.GetLogger())
+	db, err := postgresql.Builder(databaseCtx)
+	if err != nil {
+		return fmt.Errorf("error while building the db: %s", err)
+	}
+
+	// Build the migrator and perform the migrations
+	migrator := v5db.NewMigrator(db.(*postgresql.Database))
+	return migrator.Migrate()
+}

cmd/migrate/v5/utils.go

@@ -0,0 +1,31 @@
+package v5
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	utils "github.com/forbole/juno/v5/cmd/migrate/utils"
+	"gopkg.in/yaml.v3"
+
+	"github.com/forbole/juno/v5/types/config"
+)
+
+// GetConfig returns the configuration reading it from the config.yaml file present inside the home directory
+func GetConfig() (utils.Config, error) {
+	file := path.Join(config.HomePath, "config.yaml")
+
+	// Make sure the path exists
+	if _, err := os.Stat(file); os.IsNotExist(err) {
+		return utils.Config{}, fmt.Errorf("config file does not exist")
+	}
+
+	bz, err := os.ReadFile(file)
+	if err != nil {
+		return utils.Config{}, fmt.Errorf("error while reading config files: %s", err)
+	}
+
+	var cfg utils.Config
+	err = yaml.Unmarshal(bz, &cfg)
+	return cfg, err
+}

database/database.go

@@ -51,6 +51,10 @@ type Database interface {
 	// An error is returned if the operation fails.
 	SaveMessage(msg *types.Message) error
 
+	// SaveMessageType stores a single message type.
+	// An error is returned if the operation fails.
+	SaveMessageType(msg *types.MessageType) error
+
 	// Close closes the connection to the database
 	Close()
 }

database/legacy/v5/migrate.go

@@ -0,0 +1,54 @@
+package v5
+
+import (
+	"fmt"
+
+	dbtypes "github.com/forbole/juno/v5/database/migrate/utils"
+	msg "github.com/forbole/juno/v5/modules/messages"
+	"github.com/forbole/juno/v5/types"
+)
+
+// Migrate implements database.Migrator
+func (db *Migrator) Migrate() error {
+	msgTypes, err := db.getMsgTypesFromMessageTable()
+	if err != nil {
+		return fmt.Errorf("error while getting message types rows: %s", err)
+	}
+
+	for _, msgType := range msgTypes {
+		// migrate message types
+		err = db.migrateMsgTypes(types.NewMessageType(
+			msgType.Type,
+			msg.GetModuleNameFromTypeURL(msgType.Type),
+			msg.GetMsgFromTypeURL(msgType.Type),
+			msgType.Height))
+
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// getMsgTypesFromMessageTable retrieves messages types stored in database inside message table
+func (db *Migrator) getMsgTypesFromMessageTable() ([]dbtypes.MessageRow, error) {
+	smt := "SELECT DISTINCT ON (type) type, transaction_hash, height FROM message ORDER BY type DESC"
+	var rows []dbtypes.MessageRow
+	err := db.SQL.Select(&rows, smt)
+	if err != nil {
+		return nil, err
+	}
+
+	return rows, nil
+}
+
+// migrateMsgTypes stores the given message type inside the database
+func (db *Migrator) migrateMsgTypes(msg *types.MessageType) error {
+	stmt := `
+INSERT INTO message_type(type, module, label, height) 
+VALUES ($1, $2, $3, $4) 
+ON CONFLICT (type) DO NOTHING`
+
+	_, err := db.SQL.Exec(stmt, msg.Type, msg.Module, msg.Label, msg.Height)
+	return err
+}

database/legacy/v5/migrator.go

@@ -0,0 +1,21 @@
+package v5
+
+import (
+	"github.com/jmoiron/sqlx"
+
+	"github.com/forbole/juno/v5/database"
+	"github.com/forbole/juno/v5/database/postgresql"
+)
+
+var _ database.Migrator = &Migrator{}
+
+// Migrator represents the database migrator that should be used to migrate from v4 of the database to v5
+type Migrator struct {
+	SQL *sqlx.DB
+}
+
+func NewMigrator(db *postgresql.Database) *Migrator {
+	return &Migrator{
+		SQL: db.SQL,
+	}
+}

database/migrate/utils/types.go

@@ -14,3 +14,12 @@ type TransactionRow struct {
 	RawLog      string `db:"raw_log"`
 	Logs        string `db:"logs"`
 }
+
+type MessageRow struct {
+	TransactionHash           string `db:"transaction_hash"`
+	Index                     int64  `db:"index"`
+	Type                      string `db:"type"`
+	Value                     string `db:"value"`
+	InvolvedAccountsAddresses string `db:"involved_accounts_addresses"`
+	Height                    int64  `db:"height"`
+}

database/postgresql/postgresql.go

@@ -314,6 +314,17 @@ ON CONFLICT (transaction_hash, index, partition_id) DO UPDATE
 	return err
 }
 
+// SaveMessageType stores the given message type inside the database
+func (db *Database) SaveMessageType(msg *types.MessageType) error {
+	stmt := `
+INSERT INTO message_type(type, module, label, height) 
+VALUES ($1, $2, $3, $4) 
+ON CONFLICT (type) DO NOTHING`
+
+	_, err := db.SQL.Exec(stmt, msg.Type, msg.Module, msg.Label, msg.Height)
+	return err
+}
+
 // Close implements database.Database
 func (db *Database) Close() {
 	err := db.SQL.Close()

database/postgresql/schema.sql

@@ -59,11 +59,22 @@ CREATE INDEX transaction_hash_index ON transaction (hash);
 CREATE INDEX transaction_height_index ON transaction (height);
 CREATE INDEX transaction_partition_id_index ON transaction (partition_id);
 
+CREATE TABLE message_type
+(
+    type      TEXT   NOT NULL UNIQUE,
+    module    TEXT   NOT NULL,
+    label     TEXT   NOT NULL,
+    height    BIGINT NOT NULL
+);
+CREATE INDEX message_type_module_index ON message_type (module);
+CREATE INDEX message_type_type_index ON message_type (type);
+
+
 CREATE TABLE message
 (
     transaction_hash            TEXT   NOT NULL,
     index                       BIGINT NOT NULL,
-    type                        TEXT   NOT NULL,
+    type                        TEXT   NOT NULL REFERENCES message_type(type),
     value                       JSONB  NOT NULL,
     involved_accounts_addresses TEXT[] NOT NULL,
 
@@ -97,4 +108,15 @@ $$ LANGUAGE sql STABLE;
 CREATE TABLE pruning
 (
     last_pruned_height BIGINT NOT NULL
-)
+)
+
+CREATE FUNCTION messages_by_type(
+  types text [],
+  "limit" bigint DEFAULT 100,
+  "offset" bigint DEFAULT 0) 
+  RETURNS SETOF message AS 
+$$ 
+SELECT * FROM message
+WHERE (cardinality(types) = 0 OR type = ANY (types))
+ORDER BY height DESC LIMIT "limit" OFFSET "offset" 
+$$ LANGUAGE sql STABLE;

modules/messages/message_handler.go

@@ -2,6 +2,7 @@ package messages
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/cosmos/cosmos-sdk/codec"
 	sdk "github.com/cosmos/cosmos-sdk/types"
@@ -30,6 +31,17 @@ func HandleMsg(
 		return err
 	}
 
+	// Save message type
+	err = db.SaveMessageType(types.NewMessageType(
+		proto.MessageName(msg),
+		GetModuleNameFromTypeURL(proto.MessageName(msg)),
+		GetMsgFromTypeURL(proto.MessageName(msg)),
+		tx.Height))
+
+	if err != nil {
+		return err
+	}
+
 	// Handle MsgRecvPacket data object
 	if msgIBC, ok := msg.(*channeltypes.MsgRecvPacket); ok {
 		trimMessageString := TrimLastChar(string(bz))
@@ -53,3 +65,27 @@ func HandleMsg(
 		tx.Height,
 	))
 }
+
+func GetModuleNameFromTypeURL(input string) string {
+	moduleName := strings.Split(input, ".")
+	if len(moduleName) > 1 {
+		switch {
+		case strings.Contains(moduleName[0], "cosmos"):
+			return moduleName[1] // e.g. "cosmos.bank.v1beta1.MsgSend" => "bank"
+		case strings.Contains(moduleName[0], "ibc"):
+			return fmt.Sprintf("%s %s %s", moduleName[0], moduleName[1], moduleName[2]) // e.g. "ibc.core.client.v1.MsgUpdateClient" => "ibc core client"
+		default:
+			return fmt.Sprintf("%s %s", moduleName[0], moduleName[1]) // e.g. "cosmwasm.wasm.v1.MsgExecuteContract" => "cosmwasm wasm"
+		}
+	}
+
+	return ""
+}
+
+func GetMsgFromTypeURL(input string) string {
+	messageName := strings.Split(input, ".")
+	if len(messageName) > 1 {
+		return messageName[len(messageName)-1] // e.g. "cosmos.bank.v1beta1.MsgSend" => "MsgSend"
+	}
+	return ""
+}

types/cosmos.go

@@ -153,3 +153,23 @@ func NewMessage(txHash string, index int, msgType string, value string, addresse
 		Height:    height,
 	}
 }
+
+// -------------------------------------------------------------------------------------------------------------------
+
+// MessageType represents the data of a single message type
+type MessageType struct {
+	Type   string
+	Module string
+	Label  string
+	Height int64
+}
+
+// NewMessageType allows to build a new MessageType instance
+func NewMessageType(msgType string, module string, label string, height int64) *MessageType {
+	return &MessageType{
+		Type:   msgType,
+		Module: module,
+		Label:  label,
+		Height: height,
+	}
+}