Merge branch 'feature/packaging' into develop

# Conflicts:
#	manifest/signature.go

cli/flags.go → cmd/substreams/flags.go

@@ -1,4 +1,4 @@
-package cli
+package main
 
 import (
 	"fmt"

cli/logging.go → cmd/substreams/logging.go

@@ -1,4 +1,4 @@
-package cli
+package main
 
 import (
 	"github.com/streamingfast/logging"

cmd/substreams/main.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/streamingfast/substreams/cli"
+	"github.com/streamingfast/derr"
 )
 
 // Commit sha1 value, injected via go build `ldflags` at build time
@@ -17,7 +17,9 @@ var version = "dev"
 var date = ""
 
 func main() {
-	cli.Main(computeVersionString(version, commit, date))
+	rootCmd.Version = computeVersionString(version, commit, date)
+	setup()
+	derr.Check("substreams", rootCmd.Execute())
 }
 
 func computeVersionString(version, commit, date string) string {

cmd/substreams/manifest.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"github.com/streamingfast/substreams/manifest"
+)
+
+var manifestCmd = &cobra.Command{
+	Use:          "manifest",
+	SilenceUsage: true,
+}
+var manifestInfoCmd = &cobra.Command{
+	Use:          "info <manifest_file>",
+	RunE:         runManifestInfo,
+	Args:         cobra.ExactArgs(1),
+	SilenceUsage: true,
+}
+
+var manifestGraphCmd = &cobra.Command{
+	Use:          "graph <manifest_file>",
+	RunE:         runManifestGraph,
+	Args:         cobra.ExactArgs(1),
+	SilenceUsage: true,
+}
+
+func init() {
+	manifestCmd.AddCommand(manifestInfoCmd)
+	manifestCmd.AddCommand(manifestGraphCmd)
+
+	rootCmd.AddCommand(manifestCmd)
+}
+
+func runManifestInfo(cmd *cobra.Command, args []string) error {
+
+	fmt.Println("Manifest Info")
+
+	manifestPath := args[0]
+	pkg, err := manifest.New(manifestPath)
+	if err != nil {
+		return fmt.Errorf("read manifest %q: %w", manifestPath, err)
+	}
+
+	graph, err := manifest.NewModuleGraph(pkg.Modules.Modules)
+	if err != nil {
+		return fmt.Errorf("create module graph %w", err)
+	}
+
+	fmt.Println("Description:", pkg.PackageMeta[0].Doc)
+	fmt.Println("Version:", pkg.PackageMeta[0].Version)
+	fmt.Println("----")
+	for _, module := range pkg.Modules.Modules {
+		fmt.Println("module:", module.Name)
+		fmt.Println("Kind:", module.GetKind())
+		fmt.Println("Hash:", manifest.HashModuleAsString(pkg.Modules, graph, module))
+	}
+
+	return nil
+}
+
+func runManifestGraph(cmd *cobra.Command, args []string) error {
+	manifestPath := args[0]
+	pkg, err := manifest.New(manifestPath)
+	if err != nil {
+		return fmt.Errorf("read manifest %q: %w", manifestPath, err)
+	}
+
+	manifest.PrintMermaid(pkg.Modules)
+
+	return nil
+}

cmd/substreams/pack.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/streamingfast/substreams/manifest"
+	"google.golang.org/protobuf/proto"
+)
+
+var packCmd = &cobra.Command{
+	Use:          "pack <manifest_yaml> [manifest_spkg]",
+	RunE:         runPack,
+	Args:         cobra.RangeArgs(1, 2),
+	SilenceUsage: true,
+}
+
+func init() {
+	rootCmd.AddCommand(packCmd)
+}
+
+func runPack(cmd *cobra.Command, args []string) error {
+	manifestPath := args[0]
+	pkg, err := manifest.New(manifestPath)
+	if err != nil {
+		return fmt.Errorf("reading manifest %q: %w", manifestPath, err)
+	}
+
+	if _, err = manifest.NewModuleGraph(pkg.Modules.Modules); err != nil {
+		return fmt.Errorf("processing module graph %w", err)
+	}
+
+	defaultFilename := fmt.Sprintf("%s-%s.spkg", strings.Replace(pkg.PackageMeta[0].Name, "_", "-", -1), pkg.PackageMeta[0].Version)
+	cnt, err := proto.Marshal(pkg)
+	if err != nil {
+		return fmt.Errorf("marshalling package: %w", err)
+	}
+
+	if err := ioutil.WriteFile(defaultFilename, cnt, 0644); err != nil {
+		fmt.Println("")
+		return fmt.Errorf("writing %q: %w", defaultFilename, err)
+	}
+
+	fmt.Printf(`To generate bindings for your code:
+1. create a file 'buf.gen.yaml' with this content:
+
+version: v1
+plugins:
+  - name: prost  # Generate for Rust, used by your modules, or Rust client code.
+    out: gen/src
+    opt:
+      - bytes=.
+      - compile_well_known_types
+
+2. run 'buf generate %s#format=bin'
+3. See https://crates.io/crates/protoc-gen-prost for more details
+
+`, defaultFilename)
+	fmt.Printf("----------------------------------------\n")
+	fmt.Printf("Successfully wrote %q.\n", defaultFilename)
+
+	return nil
+}

cli/parallelize.go → cmd/substreams/parallelize.go

@@ -1,4 +1,4 @@
-package cli
+package main
 
 import (
 	"encoding/json"
@@ -22,12 +22,17 @@ func runParallelizeE(cmd *cobra.Command, args []string) error {
 	manifestPath := args[0]
 	streamName := args[1]
 
-	manif, err := manifest.New(manifestPath)
+	pkg, err := manifest.New(manifestPath)
 	if err != nil {
 		return fmt.Errorf("read manifest %q: %w", manifestPath, err)
 	}
 
-	stores, err := manif.Graph.StoresDownTo([]string{streamName})
+	graph, err := manifest.NewModuleGraph(pkg.Modules.Modules)
+	if err != nil {
+		return fmt.Errorf("computing module graph: %w", err)
+	}
+
+	stores, err := graph.StoresDownTo([]string{streamName})
 	res, err := json.Marshal(manifest.ModuleMarshaler(stores))
 	if err != nil {
 		return err

cli/root.go → cmd/substreams/root.go

@@ -1,4 +1,4 @@
-package cli
+package main
 
 import (
 	"github.com/spf13/cobra"