Mx 16271 indexing incoming tokens

.github/workflows/deploy-docker.yml

@@ -3,6 +3,7 @@ name: Publish Docker image
 on:
   release:
     types: [published]
+  pull_request:
 
 jobs:
   push_to_registry:
@@ -31,24 +32,25 @@ jobs:
           fi
 
       - name: Log in to Docker Hub
+        if: ${{ github.event_name == 'release' && github.event.action == 'published' }}
         uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
+        if: ${{ github.event_name == 'release' && github.event.action == 'published' }}
         id: meta
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
-          images: multiversx/elastic-indexer
+          images: multiversx/sovereign-elastic-indexer
 
       - name: Build and push Docker image
         id: push
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
           context: .
-          file: ./Dockerfile
-          push: true
+          file: ./Dockerfile-sovereign
+          push: ${{ github.event_name == 'release' && github.event.action == 'published' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-

Dockerfile-sovereign

@@ -0,0 +1,26 @@
+FROM golang:1.20.7 as builder
+
+RUN apt-get update && apt-get install -y
+
+WORKDIR /multiversx
+COPY . .
+
+WORKDIR /multiversx/cmd/elasticindexer
+
+RUN go build -o elasticindexer
+
+# ===== SECOND STAGE ======
+FROM ubuntu:22.04
+RUN apt-get update && apt-get install -y
+
+RUN useradd -m -u 1000 appuser
+USER appuser
+
+COPY --from=builder --chown=appuser /multiversx/cmd/elasticindexer /multiversx
+
+EXPOSE 22111
+
+WORKDIR /multiversx
+
+ENTRYPOINT ["./elasticindexer", "--sovereign"]
+CMD ["--log-level", "*:DEBUG"]

Makefile

@@ -36,9 +36,16 @@ integration-tests-open-search:
 INDEXER_IMAGE_NAME="elasticindexer"
 INDEXER_IMAGE_TAG="latest"
 DOCKER_FILE=Dockerfile
+SOVEREIGN_DOCKER_FILE=Dockerfile-sovereign
 
 docker-build:
 	docker build \
 		 -t ${INDEXER_IMAGE_NAME}:${INDEXER_IMAGE_TAG} \
 		 -f ${DOCKER_FILE} \
 		 .
+
+docker-sovereign-build:
+	docker build \
+		 -t ${INDEXER_IMAGE_NAME}:${INDEXER_IMAGE_TAG} \
+		 -f ${SOVEREIGN_DOCKER_FILE} \
+		 .

client/disabled/elasticClient.go

@@ -0,0 +1,78 @@
+package disabled
+
+import (
+	"bytes"
+	"context"
+)
+
+type elasticClient struct{}
+
+// NewDisabledElasticClient -
+func NewDisabledElasticClient() *elasticClient {
+	return &elasticClient{}
+}
+
+// DoBulkRequest -
+func (ec *elasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error {
+	return nil
+}
+
+// DoQueryRemove -
+func (ec *elasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error {
+	return nil
+}
+
+// DoMultiGet -
+func (ec *elasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error {
+	return nil
+}
+
+// DoScrollRequest -
+func (ec *elasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error {
+	return nil
+}
+
+// DoCountRequest -
+func (ec *elasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) {
+	return 0, nil
+}
+
+// UpdateByQuery -
+func (ec *elasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error {
+	return nil
+}
+
+// PutMappings -
+func (ec *elasticClient) PutMappings(_ string, _ *bytes.Buffer) error {
+	return nil
+}
+
+// CheckAndCreateIndex -
+func (ec *elasticClient) CheckAndCreateIndex(_ string) error {
+	return nil
+}
+
+// CheckAndCreateAlias -
+func (ec *elasticClient) CheckAndCreateAlias(_ string, _ string) error {
+	return nil
+}
+
+// CheckAndCreateTemplate -
+func (ec *elasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error {
+	return nil
+}
+
+// CheckAndCreatePolicy -
+func (ec *elasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error {
+	return nil
+}
+
+// IsEnabled -
+func (ec *elasticClient) IsEnabled() bool {
+	return false
+}
+
+// IsInterfaceNil - returns true if there is no value under the interface
+func (ec *elasticClient) IsInterfaceNil() bool {
+	return ec == nil
+}

client/disabled/elasticClient_test.go

@@ -0,0 +1,31 @@
+package disabled
+
+import (
+	"bytes"
+	"context"
+	"testing"
+
+	"github.com/multiversx/mx-chain-core-go/core/check"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDisabledElasticClient_MethodsShouldNotPanic(t *testing.T) {
+	t.Parallel()
+
+	ec := NewDisabledElasticClient()
+	require.False(t, check.IfNil(ec))
+
+	require.NotPanics(t, func() {
+		_ = ec.DoBulkRequest(context.Background(), new(bytes.Buffer), "")
+		_ = ec.DoQueryRemove(context.Background(), "", new(bytes.Buffer))
+		_ = ec.DoMultiGet(context.Background(), make([]string, 0), "", true, nil)
+		_ = ec.DoScrollRequest(context.Background(), "", []byte(""), true, nil)
+		_, _ = ec.DoCountRequest(context.Background(), "", []byte(""))
+		_ = ec.UpdateByQuery(context.Background(), "", new(bytes.Buffer))
+		_ = ec.PutMappings("", new(bytes.Buffer))
+		_ = ec.CheckAndCreateIndex("")
+		_ = ec.CheckAndCreateAlias("", "")
+		_ = ec.CheckAndCreateTemplate("", new(bytes.Buffer))
+		_ = ec.CheckAndCreatePolicy("", new(bytes.Buffer))
+	})
+}

client/elasticClientCommon.go

@@ -6,11 +6,14 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math"
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/elastic/go-elasticsearch/v7/esapi"
+
 	"github.com/multiversx/mx-chain-es-indexer-go/data"
 	"github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer"
 )
@@ -268,3 +271,11 @@ func parseResponse(res *esapi.Response, dest interface{}, errorHandler responseE
 
 	return nil
 }
+
+// RetryBackOff returns elastic retry backoff duration
+func RetryBackOff(attempt int) time.Duration {
+	d := time.Duration(math.Exp2(float64(attempt))) * time.Second
+	log.Debug("elastic: retry backoff", "attempt", attempt, "sleep duration", d)
+
+	return d
+}

client/mainChainElasticClient.go

@@ -0,0 +1,28 @@
+package client
+
+import (
+	"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc"
+)
+
+type mainChainElasticClient struct {
+	elasticproc.DatabaseClientHandler
+	indexingEnabled bool
+}
+
+// NewMainChainElasticClient creates a new sovereign elastic client
+func NewMainChainElasticClient(esClient elasticproc.DatabaseClientHandler, indexingEnabled bool) (*mainChainElasticClient, error) {
+	return &mainChainElasticClient{
+		esClient,
+		indexingEnabled,
+	}, nil
+}
+
+// IsEnabled returns true if main chain elastic client is enabled
+func (mcec *mainChainElasticClient) IsEnabled() bool {
+	return mcec.indexingEnabled
+}
+
+// IsInterfaceNil returns true if there is no value under the interface
+func (mcec *mainChainElasticClient) IsInterfaceNil() bool {
+	return mcec == nil
+}

client/mainChainElasticClient_test.go

@@ -0,0 +1,33 @@
+package client
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewMainChainElasticClient(t *testing.T) {
+	esClient, err := NewElasticClient(elasticsearch.Config{
+		Addresses: []string{"http://localhost:9200"},
+	})
+	require.Nil(t, err)
+	require.NotNil(t, esClient)
+
+	mainChainESClient, err := NewMainChainElasticClient(esClient, true)
+	require.NoError(t, err)
+	require.Equal(t, "*client.mainChainElasticClient", fmt.Sprintf("%T", mainChainESClient))
+}
+
+func TestMainChainElasticClient_IsEnabled(t *testing.T) {
+	esClient, err := NewElasticClient(elasticsearch.Config{
+		Addresses: []string{"http://localhost:9200"},
+	})
+	require.Nil(t, err)
+	require.NotNil(t, esClient)
+
+	mainChainESClient, err := NewMainChainElasticClient(esClient, true)
+	require.NoError(t, err)
+	require.Equal(t, true, mainChainESClient.IsEnabled())
+}

cmd/elasticindexer/config/config.toml

@@ -4,6 +4,7 @@
         "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags",
         "logs", "delegators", "operations", "esdts", "values", "events"
     ]
+    esdt-prefix = ""
     [config.address-converter]
         length = 32
         type = "bech32"

cmd/elasticindexer/config/prefs.toml

@@ -23,3 +23,11 @@
         username = ""
         password = ""
         bulk-request-max-size-in-bytes = 4194304 # 4MB
+
+    # Configuration for main chain elastic cluster
+    # Used by the sovereign chain indexer to index incoming new tokens properties
+    [config.main-chain-elastic-cluster]
+        enabled = true
+        url = "http://localhost:9201"
+        username = ""
+        password = ""

config/config.go

@@ -4,6 +4,7 @@ package config
 type Config struct {
 	Config struct {
 		AvailableIndices []string `toml:"available-indices"`
+		ESDTPrefix       string   `toml:"esdt-prefix"`
 		AddressConverter struct {
 			Length int    `toml:"length"`
 			Type   string `toml:"type"`
@@ -52,6 +53,12 @@ type ClusterConfig struct {
 			Password                  string `toml:"password"`
 			BulkRequestMaxSizeInBytes int    `toml:"bulk-request-max-size-in-bytes"`
 		} `toml:"elastic-cluster"`
+		MainChainCluster struct {
+			Enabled  bool   `toml:"enabled"`
+			URL      string `toml:"url"`
+			UserName string `toml:"username"`
+			Password string `toml:"password"`
+		} `toml:"main-chain-elastic-cluster"`
 	} `toml:"config"`
 }
 

data/tokens.go

@@ -45,6 +45,18 @@ type SourceToken struct {
 	CurrentOwner string `json:"currentOwner"`
 }
 
+// ResponseTokenInfo is the structure for the tokens info response
+type ResponseTokenInfo struct {
+	Docs []ResponseTokenInfoDB `json:"docs"`
+}
+
+// ResponseTokenInfoDB is the structure for the token info response
+type ResponseTokenInfoDB struct {
+	Found  bool      `json:"found"`
+	ID     string    `json:"_id"`
+	Source TokenInfo `json:"_source"`
+}
+
 // TokenInfo is a structure that is needed to store information about a token
 type TokenInfo struct {
 	Name              string           `json:"name,omitempty"`

docker-compose.yml

@@ -15,6 +15,22 @@ services:
     ports:
       - "9200:9200"
       - "9300:9300"
+  main-chain-elasticsearch:
+    container_name: es-container2
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.16.1
+    environment:
+      - "discovery.type=single-node"
+      - "xpack.security.enabled=false"
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    networks:
+      - es-net
+    ports:
+      - "9201:9200"
+      - "9301:9300"
   kibana:
     container_name: kb-container
     image: docker.elastic.co/kibana/kibana:7.16.1

factory/runType/interface.go

@@ -1,12 +1,13 @@
 package runType
 
 import (
+	"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc"
 	"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/transactions"
 )
 
 // RunTypeComponentsCreator is the interface for creating run type components
 type RunTypeComponentsCreator interface {
-	Create() *runTypeComponents
+	Create() (*runTypeComponents, error)
 	IsInterfaceNil() bool
 }
 
@@ -28,6 +29,7 @@ type RunTypeComponentsHandler interface {
 type RunTypeComponentsHolder interface {
 	TxHashExtractorCreator() transactions.TxHashExtractor
 	RewardTxDataCreator() transactions.RewardTxDataHandler
+	IndexTokensHandlerCreator() elasticproc.IndexTokensHandler
 	Create() error
 	Close() error
 	CheckSubcomponents() error