From df54de8aafdcd881aefee3dd85afe4e65a99ac57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Thu, 29 Aug 2024 12:14:15 +0200 Subject: [PATCH 1/6] Add Solana foundational module --- server/handler_convo.go | 1 + sol-transactions/convo.go | 322 ++++++++++++++++++ sol-transactions/convo_test.go | 20 ++ sol-transactions/generate.go | 223 ++++++++++++ sol-transactions/logging.go | 7 + sol-transactions/state.go | 29 ++ sol-transactions/templates/README.md.gotmpl | 46 +++ .../templates/substreams.yaml.gotmpl | 16 + sol-transactions/types.go | 6 + 9 files changed, 670 insertions(+) create mode 100644 sol-transactions/convo.go create mode 100644 sol-transactions/convo_test.go create mode 100644 sol-transactions/generate.go create mode 100644 sol-transactions/logging.go create mode 100644 sol-transactions/state.go create mode 100644 sol-transactions/templates/README.md.gotmpl create mode 100644 sol-transactions/templates/substreams.yaml.gotmpl create mode 100644 sol-transactions/types.go diff --git a/server/handler_convo.go b/server/handler_convo.go index 0c665ee..4a89273 100644 --- a/server/handler_convo.go +++ b/server/handler_convo.go @@ -27,6 +27,7 @@ import ( _ "github.com/streamingfast/substreams-codegen/injective-minimal" _ "github.com/streamingfast/substreams-codegen/sol-minimal" _ "github.com/streamingfast/substreams-codegen/starknet-minimal" + _ "github.com/streamingfast/substreams-codegen/sol-transactions" _ "github.com/streamingfast/substreams-codegen/vara-minimal" ) diff --git a/sol-transactions/convo.go b/sol-transactions/convo.go new file mode 100644 index 0000000..274c8fb --- /dev/null +++ b/sol-transactions/convo.go @@ -0,0 +1,322 @@ +package soltransactions + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strconv" + "time" + + codegen "github.com/streamingfast/substreams-codegen" + "github.com/streamingfast/substreams-codegen/loop" +) + +type Convo struct { + factory *codegen.MsgWrapFactory + state *Project + remoteBuildState *codegen.RemoteBuildState +} + +func init() { + codegen.RegisterConversation( + "sol-transactions", + "Simplest Substreams to get you started on solana", + `This creating the most simple substreams on Solana`, + codegen.ConversationFactory(New), + 100, + ) +} + +func New(factory *codegen.MsgWrapFactory) codegen.Conversation { + h := &Convo{ + state: &Project{}, + factory: factory, + remoteBuildState: &codegen.RemoteBuildState{}, + } + return h +} + +func (h *Convo) msg() *codegen.MsgWrap { return h.factory.NewMsg(h.state) } +func (h *Convo) action(element any) *codegen.MsgWrap { + return h.factory.NewInput(element, h.state) +} + +func cmd(msg any) loop.Cmd { + return func() loop.Msg { + return msg + } +} + +func (c *Convo) validate() error { + if _, err := json.Marshal(c.state); err != nil { + return fmt.Errorf("validating state format: %w", err) + } + return nil +} + +func (c *Convo) NextStep() loop.Cmd { + if err := c.validate(); err != nil { + return loop.Quit(err) + } + return c.state.NextStep() +} + +func (p *Project) NextStep() (out loop.Cmd) { + if p.Name == "" { + return cmd(codegen.AskProjectName{}) + } + + if !p.InitialBlockSet { + return cmd(codegen.AskInitialStartBlockType{}) + } + + if p.ProgramId == "" { + return cmd(AskProgramId{}) + } + + if !p.generatedCodeCompleted { + return cmd(codegen.RunGenerate{}) + } + + // Remote build part removed for the moment + // if !p.confirmDoCompile && !p.confirmDownloadOnly { + // return cmd(codegen.AskConfirmCompile{}) + // } + + return cmd(codegen.RunBuild{}) +} + +func (c *Convo) Update(msg loop.Msg) loop.Cmd { + if os.Getenv("SUBSTREAMS_DEV_DEBUG_CONVERSATION") == "true" { + fmt.Printf("convo Update message: %T %#v\n-> state: %#v\n\n", msg, msg, c.state) + } + + switch msg := msg.(type) { + case codegen.MsgStart: + var msgCmd loop.Cmd + if msg.Hydrate != nil { + if err := json.Unmarshal([]byte(msg.Hydrate.SavedState), &c.state); err != nil { + return loop.Quit(fmt.Errorf(`something went wrong, here's an error message to share with our devs (%s); we've notified them already`, err)) + } + + msgCmd = c.msg().Message("Ok, I reloaded your state.").Cmd() + } else { + msgCmd = c.msg().Message("Ok, let's start a new package.").Cmd() + } + return loop.Seq(msgCmd, c.NextStep()) + + case codegen.AskProjectName: + return c.action(codegen.InputProjectName{}). + TextInput(codegen.InputProjectNameTextInput(), "Submit"). + Description(codegen.InputProjectNameDescription()). + DefaultValue("my_project"). + Validation(codegen.InputProjectNameRegex(), codegen.InputProjectNameValidation()). + Cmd() + + case codegen.InputProjectName: + c.state.Name = msg.Value + return c.NextStep() + + case codegen.AskInitialStartBlockType: + return c.action(codegen.InputAskInitialStartBlockType{}). + TextInput(codegen.InputAskInitialStartBlockTypeTextInput(), "Submit"). + DefaultValue("0"). + Validation(codegen.InputAskInitialStartBlockTypeRegex(), codegen.InputAskInitialStartBlockTypeValidation()). + Cmd() + + case codegen.InputAskInitialStartBlockType: + initialBlock, err := strconv.ParseUint(msg.Value, 10, 64) + if err != nil { + return loop.Quit(fmt.Errorf("invalid start block input value %q, expected a number", msg.Value)) + } + + c.state.InitialBlock = initialBlock + c.state.InitialBlockSet = true + return c.NextStep() + + case AskProgramId: + return c.action(InputProgramId{}). + TextInput(fmt.Sprintf("Filter the transactions based on one or several Program IDs.\n\nSupported operators are: logical or '||', logical and '&&' and parenthesis: '()'. \n\nExample: to only consume TRANSACTIONS containing Token or ComputeBudget instructions: 'program:TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA || program:ComputeBudget111111111111111111111111111111'. \n\nTransactions containing 'Vote111111111111111111111111111111111111111' instructions are always excluded."), "Submit"). + DefaultValue("program:TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA"). + Cmd() + + case InputProgramId: + c.state.ProgramId = msg.Value + fmt.Printf("%s", msg.Value) + return c.NextStep() + + // Remote build part removed for the moment + // case codegen.InputConfirmCompile: + // if msg.Affirmative { + // c.state.confirmDoCompile = true + // } else { + // c.state.confirmDownloadOnly = true + // } + // return c.NextStep() + + case codegen.RunGenerate: + return loop.Seq( + cmdGenerate(c.state), + ) + + // Remote build part removed for the moment + // case codegen.AskConfirmCompile: + // return c.action(codegen.InputConfirmCompile{}). + // Confirm("Should we build the Substreams package for you?", "Yes, build it", "No"). + // Cmd() + + case codegen.ReturnGenerate: + if msg.Err != nil { + return loop.Seq( + c.msg().Messagef("Code generation failed with error: %s", msg.Err).Cmd(), + loop.Quit(msg.Err), + ) + } + + c.state.projectFiles = msg.ProjectFiles + c.state.generatedCodeCompleted = true + + downloadCmd := c.action(codegen.InputSourceDownloaded{}).DownloadFiles() + + for fileName, fileContent := range msg.SourceFiles { + fileDescription := "" + if _, ok := codegen.FileDescriptions[fileName]; ok { + fileDescription = codegen.FileDescriptions[fileName] + } + + downloadCmd.AddFile(fileName, fileContent, "text/plain", fileDescription) + } + + for fileName, fileContent := range msg.ProjectFiles { + fileDescription := "" + if _, ok := codegen.FileDescriptions[fileName]; ok { + fileDescription = codegen.FileDescriptions[fileName] + } + + downloadCmd.AddFile(fileName, fileContent, "text/plain", fileDescription) + } + + return loop.Seq(c.msg().Messagef("Code generation complete!").Cmd(), downloadCmd.Cmd()) + + case codegen.InputSourceDownloaded: + return c.NextStep() + + case codegen.RunBuild: + // Remote build part removed for the moment + // Do not run the build, the user only wants to download the files + // if c.state.confirmDownloadOnly { + // return cmd(codegen.ReturnBuild{ + // Err: nil, + // Artifacts: nil, + // }) + // } + + return cmd(codegen.ReturnBuild{ + Err: nil, + Artifacts: nil, + }) + + // Remote build part removed for the moment + // return cmdBuild(c.state) + + case codegen.CompilingBuild: + resp, ok := <-msg.RemoteBuildChan + + if !ok { + // the channel has been closed, we are done + return loop.Seq( + c.msg().StopLoading().Cmd(), + cmdBuildCompleted(c.remoteBuildState), + ) + } + + if resp == nil { + // dont fail the command line yet, go to the return build step + return loop.Seq( + c.msg().StopLoading().Cmd(), + cmdBuildFailed(nil, errors.New("build response is nil")), + ) + } + + if resp.Error != "" { + // dont fail the command line yet, go to the return build step + return loop.Seq( + // This is not an error, send a loading false to remove the loading spinner + c.msg().Loading(false, "").Cmd(), + cmdBuildFailed(resp.Logs, errors.New(resp.Error)), + ) + } + + c.remoteBuildState.Update(resp) + + // the first time, we want to show a message stating that we have started the build + if msg.FirstTime { + return loop.Seq( + c.msg().Loadingf(true, "Compiling your Substreams, build started at %s. This normally takes around 1 minute...", c.state.buildStarted.Format(time.UnixDate)).Cmd(), + cmd(codegen.CompilingBuild{ + FirstTime: false, + RemoteBuildChan: msg.RemoteBuildChan, + }), // keep staying in the CompilingBuild state + ) + } + + if len(resp.Artifacts) == 0 { + if len(c.remoteBuildState.Logs) == 0 { + // don't accumulate any empty logs, just keep looping + return loop.Seq( + cmd(codegen.CompilingBuild{ + FirstTime: false, + RemoteBuildChan: msg.RemoteBuildChan, + }), // keep staying in the CompilingBuild state + ) + } + + return cmd(codegen.CompilingBuild{ + FirstTime: false, + RemoteBuildChan: msg.RemoteBuildChan, + }) + } + + // done, we have the artifacts + return loop.Seq( + // This is not an error, send a loading false to remove the loading spinner + c.msg().Loading(false, "").Cmd(), + cmdBuildCompleted(c.remoteBuildState), + ) + + case codegen.ReturnBuild: + // Remote build part removed for the moment + // if msg.Err != nil { + // return loop.Seq( + // c.msg().Messagef("Remote build failed with error: %q. See full logs in `{project-path}/logs.txt`", msg.Err).Cmd(), + // c.msg().Messagef("You will need to unzip the 'substreams-src.zip' file and run `make package` to try and generate the .spkg file.").Cmd(), + // c.action(codegen.PackageDownloaded{}). + // DownloadFiles(). + // AddFile("logs.txt", []byte(msg.Logs), `text/x-logs`, ""). + // Cmd(), + // ) + // } + // if c.state.confirmDoCompile { + // return loop.Seq( + // c.msg().Messagef("Build completed successfully, took %s", time.Since(c.state.buildStarted)).Cmd(), + // c.action(codegen.PackageDownloaded{}). + // DownloadFiles(). + // // In both AddFile(...) calls, do not show any description, as we already have enough description in the substreams init part of the conversation + // AddFile(msg.Artifacts[0].Filename, msg.Artifacts[0].Content, `application/x-protobuf+sf.substreams.v1.Package`, ""). + // AddFile("logs.txt", []byte(msg.Logs), `text/x-logs`, ""). + // Cmd(), + // ) + // } + + return loop.Seq( + c.msg().Message(codegen.ReturnBuildMessage(c.state.Name)).Cmd(), + loop.Quit(nil), + ) + + case codegen.PackageDownloaded: + return loop.Quit(nil) + } + + return loop.Quit(fmt.Errorf("invalid loop message: %T", msg)) +} diff --git a/sol-transactions/convo_test.go b/sol-transactions/convo_test.go new file mode 100644 index 0000000..a994dc0 --- /dev/null +++ b/sol-transactions/convo_test.go @@ -0,0 +1,20 @@ +package soltransactions + +import ( + "testing" + + codegen "github.com/streamingfast/substreams-codegen" + "github.com/streamingfast/substreams-codegen/loop" + "github.com/stretchr/testify/assert" +) + +func TestConvoNextStep(t *testing.T) { + p := &Project{} + next := func() loop.Msg { + return p.NextStep()() + } + + assert.Equal(t, codegen.AskProjectName{}, next()) + + p.Name = "my-proj" +} diff --git a/sol-transactions/generate.go b/sol-transactions/generate.go new file mode 100644 index 0000000..44972e3 --- /dev/null +++ b/sol-transactions/generate.go @@ -0,0 +1,223 @@ +package soltransactions + +import ( + "bytes" + "context" + "embed" + "fmt" + "os" + "time" + + "strings" + + "github.com/streamingfast/dgrpc" + codegen "github.com/streamingfast/substreams-codegen" + "github.com/streamingfast/substreams-codegen/loop" + pbbuild "github.com/streamingfast/substreams-codegen/pb/sf/codegen/remotebuild/v1" + "go.uber.org/zap" +) + +//go:embed templates/* +var templatesFS embed.FS + +func cmdGenerate(p *Project) loop.Cmd { + return func() loop.Msg { + projFiles, err := p.generate() + if err != nil { + return codegen.ReturnGenerate{Err: err} + } + return codegen.ReturnGenerate{ + ProjectFiles: projFiles, + } + } +} + +func cmdBuild(p *Project) loop.Cmd { + p.buildStarted = time.Now() + p.compilingBuild = true + + return func() loop.Msg { + buildResponseChan := make(chan *codegen.RemoteBuildState, 1) + go func() { + p.build(buildResponseChan) + close(buildResponseChan) + }() + + // go to the state of compiling build + return codegen.CompilingBuild{ + FirstTime: true, + RemoteBuildChan: buildResponseChan, + } + } +} +func cmdBuildFailed(logs []string, err error) loop.Cmd { + return func() loop.Msg { + return codegen.ReturnBuild{Err: err, Logs: strings.Join(logs, "\n")} + } +} + +func cmdBuildCompleted(content *codegen.RemoteBuildState) loop.Cmd { + return func() loop.Msg { + return codegen.ReturnBuild{ + Err: nil, + Logs: strings.Join(content.Logs, "\n"), + Artifacts: content.Artifacts, + } + } +} + +func (p *Project) generate() (projFiles map[string][]byte, err error) { + // TODO: before doing any generation, we'll want to validate + // all data points that are going into source code. + // We don't want some weird things getting into `build.rs` + // and being executed server side, so we'll need pristine validation + // of all inputs here. + // TODO: add some checking to make sure `ParentContractName` of DynamicContract + // do match a Contract that exists here. + + projFiles, err = p.Render() + if err != nil { + return nil, fmt.Errorf("rendering template: %w", err) + } + + return +} + +func (p *Project) build(remoteBuildContentChan chan<- *codegen.RemoteBuildState) { + cloudRunServiceURL := "localhost:9001" + if url := os.Getenv("BUILD_SERVICE_URL"); url != "" { + cloudRunServiceURL = url + } + + plaintext := false + if strings.HasPrefix(cloudRunServiceURL, "localhost") { + plaintext = true + } + + credsOption, err := dgrpc.WithAutoTransportCredentials(false, plaintext, false) + if err != nil { + // write the error to the channel and handle it on the other side + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: err.Error(), + } + return + } + + conn, err := dgrpc.NewClientConn(cloudRunServiceURL, credsOption) + if err != nil { + // write the error to the channel and handle it on the other side + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: err.Error(), + } + return + } + + defer func() { + if err := conn.Close(); err != nil { + zlog.Error("unable to close connection gracefully", zap.Error(err)) + } + }() + + projectZip, err := codegen.ZipFiles(p.projectFiles) + if err != nil { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: err.Error(), + } + } + + client := pbbuild.NewBuildServiceClient(conn) + res, err := client.Build(context.Background(), + &pbbuild.BuildRequest{ + SourceCode: projectZip, + CollectPattern: "*.spkg", + Subfolder: "substreams", + }, + ) + + if err != nil { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: err.Error(), + } + return + } + + var aggregatedLogs []string + for { + resp, err := res.Recv() + + if resp != nil && resp.Logs != "" { + aggregatedLogs = append(aggregatedLogs, resp.Logs) + } + + if err != nil { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Logs: aggregatedLogs, + Error: err.Error(), + } + return + } + if resp == nil { + break + } + + if resp.Error != "" { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Logs: aggregatedLogs, + Error: resp.Error, + } + return + } + + if len(resp.Artifacts) != 0 { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: resp.Error, + Logs: aggregatedLogs, + Artifacts: resp.Artifacts, + } + return + } + + // send the request as we go -- not used on the client yet + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Logs: []string{resp.Logs}, + } + } +} + +// use the output type form the Project to render the templates +func (p *Project) Render() (projectFiles map[string][]byte, err error) { + projectFiles = map[string][]byte{} + + tpls, err := codegen.ParseFS(nil, templatesFS, "**/*.gotmpl") + if err != nil { + return nil, fmt.Errorf("parse templates: %w", err) + } + + templateFiles := map[string]string{ + "substreams.yaml.gotmpl": "substreams.yaml", + "README.md.gotmpl": "README.md", + // "CONTRIBUTING.md": "CONTRIBUTING.md", + } + + for templateFile, finalFileName := range templateFiles { + zlog.Debug("reading ethereum project entry", zap.String("filename", templateFile)) + + var content []byte + if strings.HasSuffix(templateFile, ".gotmpl") { + buffer := &bytes.Buffer{} + if err := tpls.ExecuteTemplate(buffer, templateFile, p); err != nil { + return nil, fmt.Errorf("embed render entry template %q: %w", templateFile, err) + } + content = buffer.Bytes() + } else { + content, err = templatesFS.ReadFile("templates/" + templateFile) + if err != nil { + return nil, fmt.Errorf("reading %q: %w", templateFile, err) + } + } + + projectFiles[finalFileName] = content + } + + return +} diff --git a/sol-transactions/logging.go b/sol-transactions/logging.go new file mode 100644 index 0000000..950bc77 --- /dev/null +++ b/sol-transactions/logging.go @@ -0,0 +1,7 @@ +package soltransactions + +import ( + "github.com/streamingfast/logging" +) + +var zlog, tracer = logging.PackageLogger("sol-transactions", "github.com/streamingfast/substreams-codegen/codegen/sol-transactions") diff --git a/sol-transactions/state.go b/sol-transactions/state.go new file mode 100644 index 0000000..52ff3a7 --- /dev/null +++ b/sol-transactions/state.go @@ -0,0 +1,29 @@ +package soltransactions + +import ( + "strings" + "time" +) + +type Project struct { + Name string `json:"name"` + ChainName string `json:"chainName"` + Compile bool `json:"compile,omitempty"` // optional field to write in state and automatically compile with no confirmation. + Download bool `json:"download,omitempty"` + InitialBlock uint64 `json:"initialBlock,omitempty"` + InitialBlockSet bool `json:"initialBlockSet,omitempty"` + ProgramId string `json:"programIdSet,omitempty"` + + // Remote build part removed for the moment + // confirmDoCompile bool + // confirmDownloadOnly bool + + generatedCodeCompleted bool + compilingBuild bool + projectFiles map[string][]byte + + buildStarted time.Time +} + +func (p *Project) ModuleName() string { return strings.ReplaceAll(p.Name, "-", "_") } +func (p *Project) KebabName() string { return strings.ReplaceAll(p.Name, "_", "-") } diff --git a/sol-transactions/templates/README.md.gotmpl b/sol-transactions/templates/README.md.gotmpl new file mode 100644 index 0000000..de97488 --- /dev/null +++ b/sol-transactions/templates/README.md.gotmpl @@ -0,0 +1,46 @@ +# Solana Transactions + +This Substreams project allows you to retrieve Solana transactions filtered by one or several Program IDs (i.e. you will only receive transactions containing the specified Program IDs). +**NOTE:** Transactions containing voting instructions will NOT be present. + +## Understand the Generated project + +Only a `substreams.yaml` file has been generated. This file declare a Substreams module, `map_filtered_transactions`, which uses the Solana Foundational Modules (module built by the team). + +```yaml +specVersion: v0.1.0 +package: + name: my_project_sol + version: v0.1.0 + +imports: + solana: https://spkg.io/streamingfast/solana-common-v0.2.0.spkg // 1. + +modules: + - name: map_filtered_transactions // 2. + use: solana:filtered_transactions_without_votes // 3. + +network: solana + +params: + map_filtered_transactions: {{ .ProgramId }} // 4. +``` +1. Import the Solana Foundational Modules. +2. Declare the `map_filtered_transactions` module, which you will run later. +3. Use the `filtered_transactions_without_votes` module from the Solana Foundational Modules. +Essentially, you are _using_ the Solana Foundational Module, which is pre-built for you. +4. Pass the regular expression to filter the transactions based on the specified Program IDs. + +## Authenticate + +To run your Substreams you will need to [authenticate](https://substreams.streamingfast.io/documentation/consume/authentication) yourself. + +```bash +substreams auth +``` + +## Run your Substreams + +```bash +substreams gui ./substreams.yaml -e mainnet.sol.streamingfast.io:443 --start-block={{ .InitialBlock }} --stop-block=+2 map_filtered_transactions +``` \ No newline at end of file diff --git a/sol-transactions/templates/substreams.yaml.gotmpl b/sol-transactions/templates/substreams.yaml.gotmpl new file mode 100644 index 0000000..6a37d94 --- /dev/null +++ b/sol-transactions/templates/substreams.yaml.gotmpl @@ -0,0 +1,16 @@ +specVersion: v0.1.0 +package: + name: my_project_sol + version: v0.1.0 + +imports: + solana: https://spkg.io/streamingfast/solana-common-v0.2.0.spkg + +modules: + - name: map_filtered_transactions + use: solana:filtered_transactions_without_votes + +network: solana + +params: + map_filtered_transactions: {{ .ProgramId }} diff --git a/sol-transactions/types.go b/sol-transactions/types.go new file mode 100644 index 0000000..00a81db --- /dev/null +++ b/sol-transactions/types.go @@ -0,0 +1,6 @@ +package soltransactions + +import pbconvo "github.com/streamingfast/substreams-codegen/pb/sf/codegen/conversation/v1" + +type AskProgramId struct{} +type InputProgramId struct{ pbconvo.UserInput_TextInput } \ No newline at end of file From dc8f750182358b321b75a28d1740e3325c6dea84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Thu, 29 Aug 2024 16:47:52 +0200 Subject: [PATCH 2/6] Update description --- sol-transactions/convo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sol-transactions/convo.go b/sol-transactions/convo.go index 274c8fb..7f15007 100644 --- a/sol-transactions/convo.go +++ b/sol-transactions/convo.go @@ -21,8 +21,8 @@ type Convo struct { func init() { codegen.RegisterConversation( "sol-transactions", - "Simplest Substreams to get you started on solana", - `This creating the most simple substreams on Solana`, + "Get Solana transactions filtered by one or several Program IDs.", + `Allows you to specified a regex containing the Program IDs used to filter the Solana transactions.`, codegen.ConversationFactory(New), 100, ) From 7c3e97fe6899a6ecc1a059b45ba0fcc020e92983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Thu, 29 Aug 2024 18:35:39 +0200 Subject: [PATCH 3/6] Fixes --- sol-transactions/state.go | 2 +- sol-transactions/templates/README.md.gotmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sol-transactions/state.go b/sol-transactions/state.go index 52ff3a7..a0be73f 100644 --- a/sol-transactions/state.go +++ b/sol-transactions/state.go @@ -12,7 +12,7 @@ type Project struct { Download bool `json:"download,omitempty"` InitialBlock uint64 `json:"initialBlock,omitempty"` InitialBlockSet bool `json:"initialBlockSet,omitempty"` - ProgramId string `json:"programIdSet,omitempty"` + ProgramId string `json:"programId,omitempty"` // Remote build part removed for the moment // confirmDoCompile bool diff --git a/sol-transactions/templates/README.md.gotmpl b/sol-transactions/templates/README.md.gotmpl index de97488..b9fa3e5 100644 --- a/sol-transactions/templates/README.md.gotmpl +++ b/sol-transactions/templates/README.md.gotmpl @@ -3,7 +3,7 @@ This Substreams project allows you to retrieve Solana transactions filtered by one or several Program IDs (i.e. you will only receive transactions containing the specified Program IDs). **NOTE:** Transactions containing voting instructions will NOT be present. -## Understand the Generated project +## Understand the Generated Project Only a `substreams.yaml` file has been generated. This file declare a Substreams module, `map_filtered_transactions`, which uses the Solana Foundational Modules (module built by the team). From 611254b5a1deb975c52d627e68506fa77331e08f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Mon, 2 Sep 2024 10:29:55 +0200 Subject: [PATCH 4/6] Fixes --- sol-transactions/convo.go | 134 +------------------- sol-transactions/templates/README.md.gotmpl | 39 +++--- sol-transactions/types.go | 3 +- 3 files changed, 28 insertions(+), 148 deletions(-) diff --git a/sol-transactions/convo.go b/sol-transactions/convo.go index 7f15007..b741ee6 100644 --- a/sol-transactions/convo.go +++ b/sol-transactions/convo.go @@ -2,11 +2,9 @@ package soltransactions import ( "encoding/json" - "errors" "fmt" "os" "strconv" - "time" codegen "github.com/streamingfast/substreams-codegen" "github.com/streamingfast/substreams-codegen/loop" @@ -79,12 +77,7 @@ func (p *Project) NextStep() (out loop.Cmd) { return cmd(codegen.RunGenerate{}) } - // Remote build part removed for the moment - // if !p.confirmDoCompile && !p.confirmDownloadOnly { - // return cmd(codegen.AskConfirmCompile{}) - // } - - return cmd(codegen.RunBuild{}) + return cmd(ShowInstructions{}) } func (c *Convo) Update(msg loop.Msg) loop.Cmd { @@ -146,26 +139,11 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { fmt.Printf("%s", msg.Value) return c.NextStep() - // Remote build part removed for the moment - // case codegen.InputConfirmCompile: - // if msg.Affirmative { - // c.state.confirmDoCompile = true - // } else { - // c.state.confirmDownloadOnly = true - // } - // return c.NextStep() - case codegen.RunGenerate: return loop.Seq( cmdGenerate(c.state), ) - // Remote build part removed for the moment - // case codegen.AskConfirmCompile: - // return c.action(codegen.InputConfirmCompile{}). - // Confirm("Should we build the Substreams package for you?", "Yes, build it", "No"). - // Cmd() - case codegen.ReturnGenerate: if msg.Err != nil { return loop.Seq( @@ -202,120 +180,12 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { case codegen.InputSourceDownloaded: return c.NextStep() - case codegen.RunBuild: - // Remote build part removed for the moment - // Do not run the build, the user only wants to download the files - // if c.state.confirmDownloadOnly { - // return cmd(codegen.ReturnBuild{ - // Err: nil, - // Artifacts: nil, - // }) - // } - - return cmd(codegen.ReturnBuild{ - Err: nil, - Artifacts: nil, - }) - - // Remote build part removed for the moment - // return cmdBuild(c.state) - - case codegen.CompilingBuild: - resp, ok := <-msg.RemoteBuildChan - - if !ok { - // the channel has been closed, we are done - return loop.Seq( - c.msg().StopLoading().Cmd(), - cmdBuildCompleted(c.remoteBuildState), - ) - } - - if resp == nil { - // dont fail the command line yet, go to the return build step - return loop.Seq( - c.msg().StopLoading().Cmd(), - cmdBuildFailed(nil, errors.New("build response is nil")), - ) - } - - if resp.Error != "" { - // dont fail the command line yet, go to the return build step - return loop.Seq( - // This is not an error, send a loading false to remove the loading spinner - c.msg().Loading(false, "").Cmd(), - cmdBuildFailed(resp.Logs, errors.New(resp.Error)), - ) - } - - c.remoteBuildState.Update(resp) - - // the first time, we want to show a message stating that we have started the build - if msg.FirstTime { - return loop.Seq( - c.msg().Loadingf(true, "Compiling your Substreams, build started at %s. This normally takes around 1 minute...", c.state.buildStarted.Format(time.UnixDate)).Cmd(), - cmd(codegen.CompilingBuild{ - FirstTime: false, - RemoteBuildChan: msg.RemoteBuildChan, - }), // keep staying in the CompilingBuild state - ) - } - - if len(resp.Artifacts) == 0 { - if len(c.remoteBuildState.Logs) == 0 { - // don't accumulate any empty logs, just keep looping - return loop.Seq( - cmd(codegen.CompilingBuild{ - FirstTime: false, - RemoteBuildChan: msg.RemoteBuildChan, - }), // keep staying in the CompilingBuild state - ) - } - - return cmd(codegen.CompilingBuild{ - FirstTime: false, - RemoteBuildChan: msg.RemoteBuildChan, - }) - } - - // done, we have the artifacts - return loop.Seq( - // This is not an error, send a loading false to remove the loading spinner - c.msg().Loading(false, "").Cmd(), - cmdBuildCompleted(c.remoteBuildState), - ) - - case codegen.ReturnBuild: - // Remote build part removed for the moment - // if msg.Err != nil { - // return loop.Seq( - // c.msg().Messagef("Remote build failed with error: %q. See full logs in `{project-path}/logs.txt`", msg.Err).Cmd(), - // c.msg().Messagef("You will need to unzip the 'substreams-src.zip' file and run `make package` to try and generate the .spkg file.").Cmd(), - // c.action(codegen.PackageDownloaded{}). - // DownloadFiles(). - // AddFile("logs.txt", []byte(msg.Logs), `text/x-logs`, ""). - // Cmd(), - // ) - // } - // if c.state.confirmDoCompile { - // return loop.Seq( - // c.msg().Messagef("Build completed successfully, took %s", time.Since(c.state.buildStarted)).Cmd(), - // c.action(codegen.PackageDownloaded{}). - // DownloadFiles(). - // // In both AddFile(...) calls, do not show any description, as we already have enough description in the substreams init part of the conversation - // AddFile(msg.Artifacts[0].Filename, msg.Artifacts[0].Content, `application/x-protobuf+sf.substreams.v1.Package`, ""). - // AddFile("logs.txt", []byte(msg.Logs), `text/x-logs`, ""). - // Cmd(), - // ) - // } - + case ShowInstructions: return loop.Seq( c.msg().Message(codegen.ReturnBuildMessage(c.state.Name)).Cmd(), loop.Quit(nil), ) - case codegen.PackageDownloaded: - return loop.Quit(nil) } return loop.Quit(fmt.Errorf("invalid loop message: %T", msg)) diff --git a/sol-transactions/templates/README.md.gotmpl b/sol-transactions/templates/README.md.gotmpl index b9fa3e5..2078b05 100644 --- a/sol-transactions/templates/README.md.gotmpl +++ b/sol-transactions/templates/README.md.gotmpl @@ -3,9 +3,32 @@ This Substreams project allows you to retrieve Solana transactions filtered by one or several Program IDs (i.e. you will only receive transactions containing the specified Program IDs). **NOTE:** Transactions containing voting instructions will NOT be present. +## Get Started + + +### Build the Substreams + +```bash +substreams build +``` + +### Authenticate + +To run your Substreams you will need to [authenticate](https://substreams.streamingfast.io/documentation/consume/authentication) yourself. + +```bash +substreams auth +``` + +### Run your Substreams + +```bash +substreams gui +``` + ## Understand the Generated Project -Only a `substreams.yaml` file has been generated. This file declare a Substreams module, `map_filtered_transactions`, which uses the Solana Foundational Modules (module built by the team). +Only a `substreams.yaml` file has been generated. This file declares a Substreams module, `map_filtered_transactions`, which uses a Solana Foundational Module (a module built by the team). ```yaml specVersion: v0.1.0 @@ -30,17 +53,3 @@ params: 3. Use the `filtered_transactions_without_votes` module from the Solana Foundational Modules. Essentially, you are _using_ the Solana Foundational Module, which is pre-built for you. 4. Pass the regular expression to filter the transactions based on the specified Program IDs. - -## Authenticate - -To run your Substreams you will need to [authenticate](https://substreams.streamingfast.io/documentation/consume/authentication) yourself. - -```bash -substreams auth -``` - -## Run your Substreams - -```bash -substreams gui ./substreams.yaml -e mainnet.sol.streamingfast.io:443 --start-block={{ .InitialBlock }} --stop-block=+2 map_filtered_transactions -``` \ No newline at end of file diff --git a/sol-transactions/types.go b/sol-transactions/types.go index 00a81db..2a979cc 100644 --- a/sol-transactions/types.go +++ b/sol-transactions/types.go @@ -3,4 +3,5 @@ package soltransactions import pbconvo "github.com/streamingfast/substreams-codegen/pb/sf/codegen/conversation/v1" type AskProgramId struct{} -type InputProgramId struct{ pbconvo.UserInput_TextInput } \ No newline at end of file +type InputProgramId struct{ pbconvo.UserInput_TextInput } +type ShowInstructions struct{} \ No newline at end of file From 624e6b110fe181555cb700709b915f0b82b6bbf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Wed, 4 Sep 2024 17:14:52 +0200 Subject: [PATCH 5/6] WIP --- evm-events-calls/convo.go | 2 +- evm-minimal/convo.go | 2 +- injective-events/convo.go | 2 +- injective-minimal/convo.go | 2 +- sol-minimal/convo.go | 2 +- sol-transactions/convo.go | 2 +- starknet-events/convo.go | 2 +- starknet-minimal/convo.go | 2 +- types.go | 22 ++++++++++++++-------- vara-minimal/convo.go | 2 +- 10 files changed, 23 insertions(+), 17 deletions(-) diff --git a/evm-events-calls/convo.go b/evm-events-calls/convo.go index 0192bc5..bce01d7 100644 --- a/evm-events-calls/convo.go +++ b/evm-events-calls/convo.go @@ -1053,7 +1053,7 @@ message {{.Proto.MessageName}} {{.Proto.OutputModuleFieldName}} { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(false)).Cmd(), loop.Quit(nil), ) diff --git a/evm-minimal/convo.go b/evm-minimal/convo.go index bbd0315..f33a674 100644 --- a/evm-minimal/convo.go +++ b/evm-minimal/convo.go @@ -336,7 +336,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(true)).Cmd(), loop.Quit(nil), ) diff --git a/injective-events/convo.go b/injective-events/convo.go index 3eac03c..523481c 100644 --- a/injective-events/convo.go +++ b/injective-events/convo.go @@ -496,7 +496,7 @@ func (c *InjectiveConvo) Update(msg loop.Msg) loop.Cmd { // Cmd(), // ) return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(false)).Cmd(), loop.Quit(nil), ) diff --git a/injective-minimal/convo.go b/injective-minimal/convo.go index e1c5bef..db2940f 100644 --- a/injective-minimal/convo.go +++ b/injective-minimal/convo.go @@ -339,7 +339,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(true)).Cmd(), loop.Quit(nil), ) diff --git a/sol-minimal/convo.go b/sol-minimal/convo.go index 4739e05..1659cf6 100644 --- a/sol-minimal/convo.go +++ b/sol-minimal/convo.go @@ -295,7 +295,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(true)).Cmd(), loop.Quit(nil), ) diff --git a/sol-transactions/convo.go b/sol-transactions/convo.go index b741ee6..1d87315 100644 --- a/sol-transactions/convo.go +++ b/sol-transactions/convo.go @@ -182,7 +182,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { case ShowInstructions: return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage(c.state.Name)).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(false)).Cmd(), loop.Quit(nil), ) diff --git a/starknet-events/convo.go b/starknet-events/convo.go index 6973be0..309424c 100644 --- a/starknet-events/convo.go +++ b/starknet-events/convo.go @@ -359,7 +359,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(false)).Cmd(), loop.Quit(nil), ) diff --git a/starknet-minimal/convo.go b/starknet-minimal/convo.go index b808abc..3f14cc8 100644 --- a/starknet-minimal/convo.go +++ b/starknet-minimal/convo.go @@ -320,7 +320,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(true)).Cmd(), loop.Quit(nil), ) diff --git a/types.go b/types.go index 3702629..6b3f2c5 100644 --- a/types.go +++ b/types.go @@ -106,14 +106,20 @@ type ReturnBuild struct { Artifacts []*pbbuild.BuildResponse_BuildArtifact } -func ReturnBuildMessage() string { +func ReturnBuildMessage(isMinimal bool) string { + var minimalStr string + + if isMinimal { + minimalStr = "* Inspect and edit the the `./src/lib.rs` file\n" + } + return cli.Dedent(fmt.Sprintf( - "Your Substreams project is ready! Follow the next steps to start streaming:\n\n" + - "* Inspect and edit the the `./lib.rs` file\n" + - "* Build it: `substreams build`\n" + - "* Authenticate: `substreams auth`\n" + - "* Stream it: `substreams gui`\n\n" + - "* Build a *Subgraph* from this substreams: `substreams codegen subgraph`\n" + + "Your Substreams project is ready! Follow the next steps to start streaming:\n\n"+ + "%s"+ + "* Build it: `substreams build`\n"+ + "* Authenticate: `substreams auth`\n"+ + "* Stream it: `substreams gui`\n\n"+ + "* Build a *Subgraph* from this substreams: `substreams codegen subgraph`\n"+ "* Feed your SQL database with this substreams: `substreams codegen sql`\n", - )) + minimalStr)) } diff --git a/vara-minimal/convo.go b/vara-minimal/convo.go index 373a0fe..e4c7692 100644 --- a/vara-minimal/convo.go +++ b/vara-minimal/convo.go @@ -320,7 +320,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(true)).Cmd(), loop.Quit(nil), ) From dab32416fdba38b38043848d2e8fcbb6d83d33fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Duchesneau?= Date: Wed, 4 Sep 2024 17:15:30 -0400 Subject: [PATCH 6/6] remove debug Printf, tidy up description of filters --- sol-transactions/convo.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sol-transactions/convo.go b/sol-transactions/convo.go index 1d87315..bd00ae7 100644 --- a/sol-transactions/convo.go +++ b/sol-transactions/convo.go @@ -130,13 +130,12 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { case AskProgramId: return c.action(InputProgramId{}). - TextInput(fmt.Sprintf("Filter the transactions based on one or several Program IDs.\n\nSupported operators are: logical or '||', logical and '&&' and parenthesis: '()'. \n\nExample: to only consume TRANSACTIONS containing Token or ComputeBudget instructions: 'program:TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA || program:ComputeBudget111111111111111111111111111111'. \n\nTransactions containing 'Vote111111111111111111111111111111111111111' instructions are always excluded."), "Submit"). + TextInput(fmt.Sprintf("Filter the transactions based on one or several Program IDs.\nSupported operators are: logical or '||', logical and '&&' and parenthesis: '()'. \nExample: to only consume TRANSACTIONS containing Token or ComputeBudget instructions: 'program:TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA || program:ComputeBudget111111111111111111111111111111'. \nTransactions containing 'Vote111111111111111111111111111111111111111' instructions are always excluded."), "Submit"). DefaultValue("program:TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA"). Cmd() case InputProgramId: c.state.ProgramId = msg.Value - fmt.Printf("%s", msg.Value) return c.NextStep() case codegen.RunGenerate: