diff --git a/evm-events-calls/convo.go b/evm-events-calls/convo.go index 0192bc5..bce01d7 100644 --- a/evm-events-calls/convo.go +++ b/evm-events-calls/convo.go @@ -1053,7 +1053,7 @@ message {{.Proto.MessageName}} {{.Proto.OutputModuleFieldName}} { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(false)).Cmd(), loop.Quit(nil), ) diff --git a/evm-minimal/convo.go b/evm-minimal/convo.go index bbd0315..f33a674 100644 --- a/evm-minimal/convo.go +++ b/evm-minimal/convo.go @@ -336,7 +336,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(true)).Cmd(), loop.Quit(nil), ) diff --git a/injective-events/convo.go b/injective-events/convo.go index 3eac03c..523481c 100644 --- a/injective-events/convo.go +++ b/injective-events/convo.go @@ -496,7 +496,7 @@ func (c *InjectiveConvo) Update(msg loop.Msg) loop.Cmd { // Cmd(), // ) return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(false)).Cmd(), loop.Quit(nil), ) diff --git a/injective-minimal/convo.go b/injective-minimal/convo.go index e1c5bef..db2940f 100644 --- a/injective-minimal/convo.go +++ b/injective-minimal/convo.go @@ -339,7 +339,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(true)).Cmd(), loop.Quit(nil), ) diff --git a/server/handler_convo.go b/server/handler_convo.go index 0c665ee..4a89273 100644 --- a/server/handler_convo.go +++ b/server/handler_convo.go @@ -27,6 +27,7 @@ import ( _ "github.com/streamingfast/substreams-codegen/injective-minimal" _ "github.com/streamingfast/substreams-codegen/sol-minimal" _ "github.com/streamingfast/substreams-codegen/starknet-minimal" + _ "github.com/streamingfast/substreams-codegen/sol-transactions" _ "github.com/streamingfast/substreams-codegen/vara-minimal" ) diff --git a/sol-minimal/convo.go b/sol-minimal/convo.go index 4739e05..1659cf6 100644 --- a/sol-minimal/convo.go +++ b/sol-minimal/convo.go @@ -295,7 +295,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(true)).Cmd(), loop.Quit(nil), ) diff --git a/sol-transactions/convo.go b/sol-transactions/convo.go new file mode 100644 index 0000000..bd00ae7 --- /dev/null +++ b/sol-transactions/convo.go @@ -0,0 +1,191 @@ +package soltransactions + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + + codegen "github.com/streamingfast/substreams-codegen" + "github.com/streamingfast/substreams-codegen/loop" +) + +type Convo struct { + factory *codegen.MsgWrapFactory + state *Project + remoteBuildState *codegen.RemoteBuildState +} + +func init() { + codegen.RegisterConversation( + "sol-transactions", + "Get Solana transactions filtered by one or several Program IDs.", + `Allows you to specified a regex containing the Program IDs used to filter the Solana transactions.`, + codegen.ConversationFactory(New), + 100, + ) +} + +func New(factory *codegen.MsgWrapFactory) codegen.Conversation { + h := &Convo{ + state: &Project{}, + factory: factory, + remoteBuildState: &codegen.RemoteBuildState{}, + } + return h +} + +func (h *Convo) msg() *codegen.MsgWrap { return h.factory.NewMsg(h.state) } +func (h *Convo) action(element any) *codegen.MsgWrap { + return h.factory.NewInput(element, h.state) +} + +func cmd(msg any) loop.Cmd { + return func() loop.Msg { + return msg + } +} + +func (c *Convo) validate() error { + if _, err := json.Marshal(c.state); err != nil { + return fmt.Errorf("validating state format: %w", err) + } + return nil +} + +func (c *Convo) NextStep() loop.Cmd { + if err := c.validate(); err != nil { + return loop.Quit(err) + } + return c.state.NextStep() +} + +func (p *Project) NextStep() (out loop.Cmd) { + if p.Name == "" { + return cmd(codegen.AskProjectName{}) + } + + if !p.InitialBlockSet { + return cmd(codegen.AskInitialStartBlockType{}) + } + + if p.ProgramId == "" { + return cmd(AskProgramId{}) + } + + if !p.generatedCodeCompleted { + return cmd(codegen.RunGenerate{}) + } + + return cmd(ShowInstructions{}) +} + +func (c *Convo) Update(msg loop.Msg) loop.Cmd { + if os.Getenv("SUBSTREAMS_DEV_DEBUG_CONVERSATION") == "true" { + fmt.Printf("convo Update message: %T %#v\n-> state: %#v\n\n", msg, msg, c.state) + } + + switch msg := msg.(type) { + case codegen.MsgStart: + var msgCmd loop.Cmd + if msg.Hydrate != nil { + if err := json.Unmarshal([]byte(msg.Hydrate.SavedState), &c.state); err != nil { + return loop.Quit(fmt.Errorf(`something went wrong, here's an error message to share with our devs (%s); we've notified them already`, err)) + } + + msgCmd = c.msg().Message("Ok, I reloaded your state.").Cmd() + } else { + msgCmd = c.msg().Message("Ok, let's start a new package.").Cmd() + } + return loop.Seq(msgCmd, c.NextStep()) + + case codegen.AskProjectName: + return c.action(codegen.InputProjectName{}). + TextInput(codegen.InputProjectNameTextInput(), "Submit"). + Description(codegen.InputProjectNameDescription()). + DefaultValue("my_project"). + Validation(codegen.InputProjectNameRegex(), codegen.InputProjectNameValidation()). + Cmd() + + case codegen.InputProjectName: + c.state.Name = msg.Value + return c.NextStep() + + case codegen.AskInitialStartBlockType: + return c.action(codegen.InputAskInitialStartBlockType{}). + TextInput(codegen.InputAskInitialStartBlockTypeTextInput(), "Submit"). + DefaultValue("0"). + Validation(codegen.InputAskInitialStartBlockTypeRegex(), codegen.InputAskInitialStartBlockTypeValidation()). + Cmd() + + case codegen.InputAskInitialStartBlockType: + initialBlock, err := strconv.ParseUint(msg.Value, 10, 64) + if err != nil { + return loop.Quit(fmt.Errorf("invalid start block input value %q, expected a number", msg.Value)) + } + + c.state.InitialBlock = initialBlock + c.state.InitialBlockSet = true + return c.NextStep() + + case AskProgramId: + return c.action(InputProgramId{}). + TextInput(fmt.Sprintf("Filter the transactions based on one or several Program IDs.\nSupported operators are: logical or '||', logical and '&&' and parenthesis: '()'. \nExample: to only consume TRANSACTIONS containing Token or ComputeBudget instructions: 'program:TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA || program:ComputeBudget111111111111111111111111111111'. \nTransactions containing 'Vote111111111111111111111111111111111111111' instructions are always excluded."), "Submit"). + DefaultValue("program:TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA"). + Cmd() + + case InputProgramId: + c.state.ProgramId = msg.Value + return c.NextStep() + + case codegen.RunGenerate: + return loop.Seq( + cmdGenerate(c.state), + ) + + case codegen.ReturnGenerate: + if msg.Err != nil { + return loop.Seq( + c.msg().Messagef("Code generation failed with error: %s", msg.Err).Cmd(), + loop.Quit(msg.Err), + ) + } + + c.state.projectFiles = msg.ProjectFiles + c.state.generatedCodeCompleted = true + + downloadCmd := c.action(codegen.InputSourceDownloaded{}).DownloadFiles() + + for fileName, fileContent := range msg.SourceFiles { + fileDescription := "" + if _, ok := codegen.FileDescriptions[fileName]; ok { + fileDescription = codegen.FileDescriptions[fileName] + } + + downloadCmd.AddFile(fileName, fileContent, "text/plain", fileDescription) + } + + for fileName, fileContent := range msg.ProjectFiles { + fileDescription := "" + if _, ok := codegen.FileDescriptions[fileName]; ok { + fileDescription = codegen.FileDescriptions[fileName] + } + + downloadCmd.AddFile(fileName, fileContent, "text/plain", fileDescription) + } + + return loop.Seq(c.msg().Messagef("Code generation complete!").Cmd(), downloadCmd.Cmd()) + + case codegen.InputSourceDownloaded: + return c.NextStep() + + case ShowInstructions: + return loop.Seq( + c.msg().Message(codegen.ReturnBuildMessage(false)).Cmd(), + loop.Quit(nil), + ) + + } + + return loop.Quit(fmt.Errorf("invalid loop message: %T", msg)) +} diff --git a/sol-transactions/convo_test.go b/sol-transactions/convo_test.go new file mode 100644 index 0000000..a994dc0 --- /dev/null +++ b/sol-transactions/convo_test.go @@ -0,0 +1,20 @@ +package soltransactions + +import ( + "testing" + + codegen "github.com/streamingfast/substreams-codegen" + "github.com/streamingfast/substreams-codegen/loop" + "github.com/stretchr/testify/assert" +) + +func TestConvoNextStep(t *testing.T) { + p := &Project{} + next := func() loop.Msg { + return p.NextStep()() + } + + assert.Equal(t, codegen.AskProjectName{}, next()) + + p.Name = "my-proj" +} diff --git a/sol-transactions/generate.go b/sol-transactions/generate.go new file mode 100644 index 0000000..44972e3 --- /dev/null +++ b/sol-transactions/generate.go @@ -0,0 +1,223 @@ +package soltransactions + +import ( + "bytes" + "context" + "embed" + "fmt" + "os" + "time" + + "strings" + + "github.com/streamingfast/dgrpc" + codegen "github.com/streamingfast/substreams-codegen" + "github.com/streamingfast/substreams-codegen/loop" + pbbuild "github.com/streamingfast/substreams-codegen/pb/sf/codegen/remotebuild/v1" + "go.uber.org/zap" +) + +//go:embed templates/* +var templatesFS embed.FS + +func cmdGenerate(p *Project) loop.Cmd { + return func() loop.Msg { + projFiles, err := p.generate() + if err != nil { + return codegen.ReturnGenerate{Err: err} + } + return codegen.ReturnGenerate{ + ProjectFiles: projFiles, + } + } +} + +func cmdBuild(p *Project) loop.Cmd { + p.buildStarted = time.Now() + p.compilingBuild = true + + return func() loop.Msg { + buildResponseChan := make(chan *codegen.RemoteBuildState, 1) + go func() { + p.build(buildResponseChan) + close(buildResponseChan) + }() + + // go to the state of compiling build + return codegen.CompilingBuild{ + FirstTime: true, + RemoteBuildChan: buildResponseChan, + } + } +} +func cmdBuildFailed(logs []string, err error) loop.Cmd { + return func() loop.Msg { + return codegen.ReturnBuild{Err: err, Logs: strings.Join(logs, "\n")} + } +} + +func cmdBuildCompleted(content *codegen.RemoteBuildState) loop.Cmd { + return func() loop.Msg { + return codegen.ReturnBuild{ + Err: nil, + Logs: strings.Join(content.Logs, "\n"), + Artifacts: content.Artifacts, + } + } +} + +func (p *Project) generate() (projFiles map[string][]byte, err error) { + // TODO: before doing any generation, we'll want to validate + // all data points that are going into source code. + // We don't want some weird things getting into `build.rs` + // and being executed server side, so we'll need pristine validation + // of all inputs here. + // TODO: add some checking to make sure `ParentContractName` of DynamicContract + // do match a Contract that exists here. + + projFiles, err = p.Render() + if err != nil { + return nil, fmt.Errorf("rendering template: %w", err) + } + + return +} + +func (p *Project) build(remoteBuildContentChan chan<- *codegen.RemoteBuildState) { + cloudRunServiceURL := "localhost:9001" + if url := os.Getenv("BUILD_SERVICE_URL"); url != "" { + cloudRunServiceURL = url + } + + plaintext := false + if strings.HasPrefix(cloudRunServiceURL, "localhost") { + plaintext = true + } + + credsOption, err := dgrpc.WithAutoTransportCredentials(false, plaintext, false) + if err != nil { + // write the error to the channel and handle it on the other side + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: err.Error(), + } + return + } + + conn, err := dgrpc.NewClientConn(cloudRunServiceURL, credsOption) + if err != nil { + // write the error to the channel and handle it on the other side + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: err.Error(), + } + return + } + + defer func() { + if err := conn.Close(); err != nil { + zlog.Error("unable to close connection gracefully", zap.Error(err)) + } + }() + + projectZip, err := codegen.ZipFiles(p.projectFiles) + if err != nil { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: err.Error(), + } + } + + client := pbbuild.NewBuildServiceClient(conn) + res, err := client.Build(context.Background(), + &pbbuild.BuildRequest{ + SourceCode: projectZip, + CollectPattern: "*.spkg", + Subfolder: "substreams", + }, + ) + + if err != nil { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: err.Error(), + } + return + } + + var aggregatedLogs []string + for { + resp, err := res.Recv() + + if resp != nil && resp.Logs != "" { + aggregatedLogs = append(aggregatedLogs, resp.Logs) + } + + if err != nil { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Logs: aggregatedLogs, + Error: err.Error(), + } + return + } + if resp == nil { + break + } + + if resp.Error != "" { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Logs: aggregatedLogs, + Error: resp.Error, + } + return + } + + if len(resp.Artifacts) != 0 { + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Error: resp.Error, + Logs: aggregatedLogs, + Artifacts: resp.Artifacts, + } + return + } + + // send the request as we go -- not used on the client yet + remoteBuildContentChan <- &codegen.RemoteBuildState{ + Logs: []string{resp.Logs}, + } + } +} + +// use the output type form the Project to render the templates +func (p *Project) Render() (projectFiles map[string][]byte, err error) { + projectFiles = map[string][]byte{} + + tpls, err := codegen.ParseFS(nil, templatesFS, "**/*.gotmpl") + if err != nil { + return nil, fmt.Errorf("parse templates: %w", err) + } + + templateFiles := map[string]string{ + "substreams.yaml.gotmpl": "substreams.yaml", + "README.md.gotmpl": "README.md", + // "CONTRIBUTING.md": "CONTRIBUTING.md", + } + + for templateFile, finalFileName := range templateFiles { + zlog.Debug("reading ethereum project entry", zap.String("filename", templateFile)) + + var content []byte + if strings.HasSuffix(templateFile, ".gotmpl") { + buffer := &bytes.Buffer{} + if err := tpls.ExecuteTemplate(buffer, templateFile, p); err != nil { + return nil, fmt.Errorf("embed render entry template %q: %w", templateFile, err) + } + content = buffer.Bytes() + } else { + content, err = templatesFS.ReadFile("templates/" + templateFile) + if err != nil { + return nil, fmt.Errorf("reading %q: %w", templateFile, err) + } + } + + projectFiles[finalFileName] = content + } + + return +} diff --git a/sol-transactions/logging.go b/sol-transactions/logging.go new file mode 100644 index 0000000..950bc77 --- /dev/null +++ b/sol-transactions/logging.go @@ -0,0 +1,7 @@ +package soltransactions + +import ( + "github.com/streamingfast/logging" +) + +var zlog, tracer = logging.PackageLogger("sol-transactions", "github.com/streamingfast/substreams-codegen/codegen/sol-transactions") diff --git a/sol-transactions/state.go b/sol-transactions/state.go new file mode 100644 index 0000000..a0be73f --- /dev/null +++ b/sol-transactions/state.go @@ -0,0 +1,29 @@ +package soltransactions + +import ( + "strings" + "time" +) + +type Project struct { + Name string `json:"name"` + ChainName string `json:"chainName"` + Compile bool `json:"compile,omitempty"` // optional field to write in state and automatically compile with no confirmation. + Download bool `json:"download,omitempty"` + InitialBlock uint64 `json:"initialBlock,omitempty"` + InitialBlockSet bool `json:"initialBlockSet,omitempty"` + ProgramId string `json:"programId,omitempty"` + + // Remote build part removed for the moment + // confirmDoCompile bool + // confirmDownloadOnly bool + + generatedCodeCompleted bool + compilingBuild bool + projectFiles map[string][]byte + + buildStarted time.Time +} + +func (p *Project) ModuleName() string { return strings.ReplaceAll(p.Name, "-", "_") } +func (p *Project) KebabName() string { return strings.ReplaceAll(p.Name, "_", "-") } diff --git a/sol-transactions/templates/README.md.gotmpl b/sol-transactions/templates/README.md.gotmpl new file mode 100644 index 0000000..2078b05 --- /dev/null +++ b/sol-transactions/templates/README.md.gotmpl @@ -0,0 +1,55 @@ +# Solana Transactions + +This Substreams project allows you to retrieve Solana transactions filtered by one or several Program IDs (i.e. you will only receive transactions containing the specified Program IDs). +**NOTE:** Transactions containing voting instructions will NOT be present. + +## Get Started + + +### Build the Substreams + +```bash +substreams build +``` + +### Authenticate + +To run your Substreams you will need to [authenticate](https://substreams.streamingfast.io/documentation/consume/authentication) yourself. + +```bash +substreams auth +``` + +### Run your Substreams + +```bash +substreams gui +``` + +## Understand the Generated Project + +Only a `substreams.yaml` file has been generated. This file declares a Substreams module, `map_filtered_transactions`, which uses a Solana Foundational Module (a module built by the team). + +```yaml +specVersion: v0.1.0 +package: + name: my_project_sol + version: v0.1.0 + +imports: + solana: https://spkg.io/streamingfast/solana-common-v0.2.0.spkg // 1. + +modules: + - name: map_filtered_transactions // 2. + use: solana:filtered_transactions_without_votes // 3. + +network: solana + +params: + map_filtered_transactions: {{ .ProgramId }} // 4. +``` +1. Import the Solana Foundational Modules. +2. Declare the `map_filtered_transactions` module, which you will run later. +3. Use the `filtered_transactions_without_votes` module from the Solana Foundational Modules. +Essentially, you are _using_ the Solana Foundational Module, which is pre-built for you. +4. Pass the regular expression to filter the transactions based on the specified Program IDs. diff --git a/sol-transactions/templates/substreams.yaml.gotmpl b/sol-transactions/templates/substreams.yaml.gotmpl new file mode 100644 index 0000000..6a37d94 --- /dev/null +++ b/sol-transactions/templates/substreams.yaml.gotmpl @@ -0,0 +1,16 @@ +specVersion: v0.1.0 +package: + name: my_project_sol + version: v0.1.0 + +imports: + solana: https://spkg.io/streamingfast/solana-common-v0.2.0.spkg + +modules: + - name: map_filtered_transactions + use: solana:filtered_transactions_without_votes + +network: solana + +params: + map_filtered_transactions: {{ .ProgramId }} diff --git a/sol-transactions/types.go b/sol-transactions/types.go new file mode 100644 index 0000000..2a979cc --- /dev/null +++ b/sol-transactions/types.go @@ -0,0 +1,7 @@ +package soltransactions + +import pbconvo "github.com/streamingfast/substreams-codegen/pb/sf/codegen/conversation/v1" + +type AskProgramId struct{} +type InputProgramId struct{ pbconvo.UserInput_TextInput } +type ShowInstructions struct{} \ No newline at end of file diff --git a/starknet-events/convo.go b/starknet-events/convo.go index 6973be0..309424c 100644 --- a/starknet-events/convo.go +++ b/starknet-events/convo.go @@ -359,7 +359,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(false)).Cmd(), loop.Quit(nil), ) diff --git a/starknet-minimal/convo.go b/starknet-minimal/convo.go index b808abc..3f14cc8 100644 --- a/starknet-minimal/convo.go +++ b/starknet-minimal/convo.go @@ -320,7 +320,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(true)).Cmd(), loop.Quit(nil), ) diff --git a/types.go b/types.go index 3702629..6b3f2c5 100644 --- a/types.go +++ b/types.go @@ -106,14 +106,20 @@ type ReturnBuild struct { Artifacts []*pbbuild.BuildResponse_BuildArtifact } -func ReturnBuildMessage() string { +func ReturnBuildMessage(isMinimal bool) string { + var minimalStr string + + if isMinimal { + minimalStr = "* Inspect and edit the the `./src/lib.rs` file\n" + } + return cli.Dedent(fmt.Sprintf( - "Your Substreams project is ready! Follow the next steps to start streaming:\n\n" + - "* Inspect and edit the the `./lib.rs` file\n" + - "* Build it: `substreams build`\n" + - "* Authenticate: `substreams auth`\n" + - "* Stream it: `substreams gui`\n\n" + - "* Build a *Subgraph* from this substreams: `substreams codegen subgraph`\n" + + "Your Substreams project is ready! Follow the next steps to start streaming:\n\n"+ + "%s"+ + "* Build it: `substreams build`\n"+ + "* Authenticate: `substreams auth`\n"+ + "* Stream it: `substreams gui`\n\n"+ + "* Build a *Subgraph* from this substreams: `substreams codegen subgraph`\n"+ "* Feed your SQL database with this substreams: `substreams codegen sql`\n", - )) + minimalStr)) } diff --git a/vara-minimal/convo.go b/vara-minimal/convo.go index 373a0fe..e4c7692 100644 --- a/vara-minimal/convo.go +++ b/vara-minimal/convo.go @@ -320,7 +320,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { // } return loop.Seq( - c.msg().Message(codegen.ReturnBuildMessage()).Cmd(), + c.msg().Message(codegen.ReturnBuildMessage(true)).Cmd(), loop.Quit(nil), )