-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This change adds the ability of scanning log messages and incrementing counters everytime a log message matches a given regular expression.
- Loading branch information
Showing
25 changed files
with
1,143 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,315 @@ | ||
| // Copyright 2022 Cockroach Labs Inc. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| // Package scan defines the sub command to run visus scanner utilities. | ||
| package scan | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "bytes" | ||
| "errors" | ||
| "fmt" | ||
| "os" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/cockroachlabs/visus/internal/database" | ||
| "github.com/cockroachlabs/visus/internal/scanner" | ||
| "github.com/cockroachlabs/visus/internal/store" | ||
| "github.com/creasty/defaults" | ||
| "github.com/prometheus/client_golang/prometheus" | ||
| "github.com/prometheus/common/expfmt" | ||
| log "github.com/sirupsen/logrus" | ||
| "github.com/spf13/cobra" | ||
| "gopkg.in/yaml.v3" | ||
| ) | ||
|
|
||
| var databaseURL = "" | ||
|
|
||
| // patternDef yaml pattern definition | ||
| type patternDef struct { | ||
| Name string | ||
| Regex string | ||
| Help string | ||
| } | ||
|
|
||
| // config yaml scanner definition | ||
| type config struct { | ||
| Name string | ||
| Path string | ||
| Format string | ||
| Enabled bool | ||
| Patterns []patternDef | ||
| } | ||
|
|
||
| func marshal(logFile *store.Scan) ([]byte, error) { | ||
| patterns := make([]patternDef, 0) | ||
| for _, m := range logFile.Patterns { | ||
| metric := patternDef{ | ||
| Name: m.Name, | ||
| Regex: m.Regex, | ||
| Help: m.Help, | ||
| } | ||
| patterns = append(patterns, metric) | ||
| } | ||
| config := &config{ | ||
| Name: logFile.Name, | ||
| Path: logFile.Path, | ||
| Format: string(logFile.Format), | ||
| Enabled: logFile.Enabled, | ||
| Patterns: patterns, | ||
| } | ||
| return yaml.Marshal(config) | ||
| } | ||
|
|
||
| // listCmd list all the log scanners in the datababse | ||
| func listCmd(factory database.Factory) *cobra.Command { | ||
| c := &cobra.Command{ | ||
| Use: "list", | ||
| Example: `./visus log list --url "postgresql://root@localhost:26257/defaultdb?sslmode=disable" `, | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| ctx := cmd.Context() | ||
| conn, err := factory.New(ctx, databaseURL) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| store := store.New(conn) | ||
| logs, err := store.GetScanNames(ctx) | ||
| if err != nil { | ||
| fmt.Print("Error retrieving log targets") | ||
| return err | ||
| } | ||
| for _, lg := range logs { | ||
| fmt.Printf("%s\n", lg) | ||
| } | ||
| return nil | ||
| }, | ||
| } | ||
| return c | ||
| } | ||
|
|
||
| func getCmd(factory database.Factory) *cobra.Command { | ||
| c := &cobra.Command{ | ||
| Use: "get", | ||
| Args: cobra.ExactArgs(1), | ||
| Example: `./visus log get scanner_name --url "postgresql://root@localhost:26257/defaultdb?sslmode=disable" `, | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| ctx := cmd.Context() | ||
| logName := args[0] | ||
| conn, err := factory.New(ctx, databaseURL) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| store := store.New(conn) | ||
| scanner, err := store.GetScan(ctx, logName) | ||
| if err != nil { | ||
| fmt.Printf("Error retrieving log %s.", logName) | ||
| return err | ||
| } | ||
| if scanner == nil { | ||
| fmt.Printf("Collection %s not found\n", logName) | ||
| } else { | ||
| res, err := marshal(scanner) | ||
| if err != nil { | ||
| fmt.Printf("Unabled to marshall %s\n", logName) | ||
| } | ||
| fmt.Println(string(res)) | ||
| } | ||
| return nil | ||
| }, | ||
| } | ||
| return c | ||
| } | ||
|
|
||
| func testCmd(factory database.Factory) *cobra.Command { | ||
| var interval time.Duration | ||
| var count int | ||
| c := &cobra.Command{ | ||
| Use: "test", | ||
| Args: cobra.ExactArgs(1), | ||
| Example: `./visus log test scanner_name --url "postgresql://root@localhost:26257/defaultdb?sslmode=disable" `, | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| ctx := cmd.Context() | ||
| logName := args[0] | ||
| conn, err := factory.ReadOnly(ctx, databaseURL) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| st := store.New(conn) | ||
| lg, err := st.GetScan(ctx, logName) | ||
| if err != nil { | ||
| fmt.Printf("Error retrieving scanner %s.", logName) | ||
| return err | ||
| } | ||
| if lg == nil { | ||
| fmt.Printf("Collection %s not found\n", logName) | ||
| } else { | ||
|
|
||
| scanner, err := scanner.New(ctx, lg, | ||
| &scanner.Config{ | ||
| FromBeginning: true, | ||
| Pool: true, | ||
| Follow: false, | ||
| }, | ||
| prometheus.DefaultRegisterer) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| scanner.Start(ctx) | ||
| for i := 1; i <= count || count == 0; i++ { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return ctx.Err() | ||
| case <-time.After(interval): | ||
| } | ||
| gathering, err := prometheus.DefaultGatherer.Gather() | ||
| if err != nil { | ||
| fmt.Printf("Error collecting patterns %s.", err) | ||
| return err | ||
| } | ||
| fmt.Printf("\n---- %s %s -----\n", time.Now().Format("01-02-2006 15:04:05"), lg.Name) | ||
| for _, mf := range gathering { | ||
| if strings.HasPrefix(*mf.Name, logName) { | ||
| expfmt.MetricFamilyToText(os.Stdout, mf) | ||
|
|
||
| } | ||
| } | ||
| } | ||
| return scanner.Stop() | ||
| } | ||
| return nil | ||
| }, | ||
| } | ||
| f := c.Flags() | ||
| f.DurationVar(&interval, "interval", 10*time.Second, "interval of scanner") | ||
| f.IntVar(&count, "count", 1, "number of times to run the scanner. Specify 0 for continuos scanner") | ||
| return c | ||
| } | ||
|
|
||
| func deleteCmd(factory database.Factory) *cobra.Command { | ||
| c := &cobra.Command{ | ||
| Use: "delete", | ||
| Args: cobra.ExactArgs(1), | ||
| Example: `./visus scanner delete scanner_name --url "postgresql://root@localhost:26257/defaultdb?sslmode=disable" `, | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| ctx := cmd.Context() | ||
| logName := args[0] | ||
| conn, err := factory.New(ctx, databaseURL) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| store := store.New(conn) | ||
| err = store.DeleteCollection(ctx, args[0]) | ||
| if err != nil { | ||
| fmt.Printf("Error deleting scanner %s.\n", logName) | ||
| return err | ||
| } | ||
| fmt.Printf("Collection %s deleted.\n", logName) | ||
| return nil | ||
| }, | ||
| } | ||
| return c | ||
| } | ||
|
|
||
| func putCmd(factory database.Factory) *cobra.Command { | ||
| var file string | ||
| c := &cobra.Command{ | ||
| Use: "put", | ||
| Args: cobra.ExactArgs(0), | ||
| Example: `./visus scanner put --yaml config.yaml --url "postgresql://root@localhost:26257/defaultdb?sslmode=disable" `, | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| ctx := cmd.Context() | ||
| if file == "" { | ||
| return errors.New("yaml configuration required") | ||
| } | ||
| conn, err := factory.New(ctx, databaseURL) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| var data []byte | ||
| if file == "-" { | ||
| var buffer bytes.Buffer | ||
| scanner := bufio.NewScanner(os.Stdin) | ||
| for scanner.Scan() { | ||
| buffer.Write(scanner.Bytes()) | ||
| buffer.WriteString("\n") | ||
| } | ||
| if err := scanner.Err(); err != nil { | ||
| log.Errorf("reading standard input: %s", err.Error()) | ||
| } | ||
| data = buffer.Bytes() | ||
| } else { | ||
| data, err = os.ReadFile(file) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| config := &config{} | ||
| err = yaml.Unmarshal(data, &config) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if err := defaults.Set(config); err != nil { | ||
| return err | ||
| } | ||
| patterns := make([]store.Pattern, 0) | ||
| for _, p := range config.Patterns { | ||
| pattern := store.Pattern{ | ||
| Name: p.Name, | ||
| Regex: p.Regex, | ||
| Help: p.Help, | ||
| } | ||
| patterns = append(patterns, pattern) | ||
| } | ||
| if config.Name == "" { | ||
| return errors.New("name must be specified") | ||
| } | ||
| logTarget := &store.Scan{ | ||
| Enabled: config.Enabled, | ||
| Format: store.LogFormat(config.Format), | ||
| Path: config.Path, | ||
| Name: config.Name, | ||
| Patterns: patterns, | ||
| } | ||
| store := store.New(conn) | ||
| err = store.PutScan(ctx, logTarget) | ||
| if err != nil { | ||
| fmt.Printf("Error inserting scanner %s.", config.Name) | ||
| return err | ||
| } | ||
| fmt.Printf("Collection %s inserted.\n", config.Name) | ||
| return nil | ||
| }, | ||
| } | ||
| f := c.Flags() | ||
| f.StringVar(&file, "yaml", "", "file containing the configuration") | ||
| return c | ||
| } | ||
|
|
||
| // Command runs the scanner tools to view and manage the configuration in the database. | ||
| func Command() *cobra.Command { | ||
| c := &cobra.Command{ | ||
| Use: "scan", | ||
| } | ||
| f := c.PersistentFlags() | ||
| c.AddCommand( | ||
| getCmd(database.DefaultFactory), | ||
| listCmd(database.DefaultFactory), | ||
| deleteCmd(database.DefaultFactory), | ||
| putCmd(database.DefaultFactory), | ||
| testCmd(database.DefaultFactory)) | ||
| f.StringVar(&databaseURL, "url", "", | ||
| "Connection URL, of the form: postgresql://[user[:passwd]@]host[:port]/[db][?parameters...]") | ||
| return c | ||
| } |
Oops, something went wrong.