CTI helpers with mitre classification etc.

.gitignore

@@ -42,6 +42,7 @@ vendor.tgz
 cmd/crowdsec-cli/cscli
 cmd/crowdsec/crowdsec
 cmd/notification-*/notification-*
+cmd/cscti/cscti
 
 # Test cache (downloaded files)
 .cache

.golangci.yml

@@ -4,6 +4,8 @@ run:
   skip-dirs:
     - pkg/time/rate
   skip-files:
+    - pkg/cti/client.go
+    - pkg/cti/types.go
     - pkg/database/ent/generate.go
     - pkg/yamlpatch/merge.go
     - pkg/yamlpatch/merge_test.go

Makefile

@@ -38,6 +38,7 @@ BUILD_CODENAME ?= alphaga
 
 CROWDSEC_FOLDER = ./cmd/crowdsec
 CSCLI_FOLDER = ./cmd/crowdsec-cli/
+CSCTI_FOLDER = ./cmd/cscti/
 PLUGINS_DIR_PREFIX = ./cmd/notification-
 
 CROWDSEC_BIN = crowdsec$(EXT)
@@ -198,6 +199,10 @@ clean: clean-debian clean-rpm testclean  ## Remove build artifacts
 cscli: goversion  ## Build cscli
 	@$(MAKE) -C $(CSCLI_FOLDER) build $(MAKE_FLAGS)
 
+.PHONY: cscti
+cscti: goversion  ## Build cscti
+	@$(MAKE) -C $(CSCTI_FOLDER) build $(MAKE_FLAGS)
+
 .PHONY: crowdsec
 crowdsec: goversion  ## Build crowdsec
 	@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
@@ -206,6 +211,8 @@ crowdsec: goversion  ## Build crowdsec
 generate:  ## Generate code for the database and APIs
 	$(GO) generate ./pkg/database/ent
 	$(GO) generate ./pkg/models
+	$(GO) generate ./pkg/cti
+	@echo "Code generation complete."
 
 .PHONY: testclean
 testclean: bats-clean  ## Remove test artifacts

cmd/crowdsec/serve.go

@@ -334,7 +334,8 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error {
 		log.Warningln("Exprhelpers loaded without database client.")
 	}
 
-	if cConfig.API.CTI != nil && *cConfig.API.CTI.Enabled {
+	// XXX: just pass the CTICfg
+	if cConfig.API.CTI != nil && cConfig.API.CTI.Enabled != nil && *cConfig.API.CTI.Enabled {
 		log.Infof("Crowdsec CTI helper enabled")
 
 		if err := exprhelpers.InitCrowdsecCTI(cConfig.API.CTI.Key, cConfig.API.CTI.CacheTimeout, cConfig.API.CTI.CacheSize, cConfig.API.CTI.LogLevel); err != nil {

cmd/cscti/Makefile

@@ -0,0 +1,32 @@
+ifeq ($(OS), Windows_NT)
+	SHELL := pwsh.exe
+	.SHELLFLAGS := -NoProfile -Command
+	EXT = .exe
+endif
+
+GO = go
+GOBUILD = $(GO) build
+
+BINARY_NAME = cscti$(EXT)
+PREFIX ?= "/"
+BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
+
+.PHONY: all
+all: clean build
+
+build: clean
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
+
+.PHONY: install
+install: install-conf install-bin
+
+install-conf:
+
+install-bin:
+	@install -v -m 755 -D "$(BINARY_NAME)" "$(BIN_PREFIX)/$(BINARY_NAME)" || exit
+
+uninstall:
+	@$(RM) $(BIN_PREFIX)$(BINARY_NAME) $(WIN_IGNORE_ERR)
+
+clean:
+	@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)

cmd/cscti/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"errors"
+	"os"
+
+	"github.com/fatih/color"
+	cc "github.com/ivanpirog/coloredcobra"
+	"github.com/spf13/cobra"
+)
+
+var (
+	ErrorNoAPIKey = errors.New("CTI_API_KEY is not set")
+)
+
+type Config struct {
+	API struct {
+		CTI struct {
+			Key string `yaml:"key"`
+		} `yaml:"cti"`
+	} `yaml:"api"`
+}
+
+func main() {
+	var configPath string
+
+	cmd := &cobra.Command{
+		Use: "cscti",
+		Short: "cscti is a tool to query the CrowdSec CTI",
+		ValidArgs: []string{"smoke-ip"},
+		DisableAutoGenTag: true,
+	}
+
+	cc.Init(&cc.Config{
+		RootCmd:       cmd,
+		Headings:      cc.Yellow,
+		Commands:      cc.Green + cc.Bold,
+		CmdShortDescr: cc.Cyan,
+		Example:       cc.Italic,
+		ExecName:      cc.Bold,
+		Aliases:       cc.Bold + cc.Italic,
+		FlagsDataType: cc.White,
+		Flags:         cc.Green,
+		FlagsDescr:    cc.Cyan,
+	})
+	cmd.SetOut(color.Output)
+
+	pflags := cmd.PersistentFlags()
+
+	pflags.StringVarP(&configPath, "config", "c", "", "Path to the configuration file")
+
+	cmd.AddCommand(NewCLISmokeIP().NewCommand())
+
+	if err := cmd.Execute(); err != nil {
+		color.Red(err.Error())
+		os.Exit(1)
+	}
+}

cmd/cscti/smokeip.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cti"
+)
+
+type cliSmokeIP struct {}
+
+func NewCLISmokeIP() *cliSmokeIP {
+	return &cliSmokeIP{}
+}
+
+func (cli *cliSmokeIP) smokeip(ip string) error {
+	// check if CTI_API_KEY is set
+	apiKey := os.Getenv("CTI_API_KEY")
+	if apiKey == "" {
+		return ErrorNoAPIKey
+	}
+
+	provider, err := cti.NewAPIKeyProvider(apiKey)
+	if err != nil {
+		return err
+	}
+
+	// create a new CTI client
+	client, err := cti.NewClientWithResponses("https://cti.api.crowdsec.net/v2/", cti.WithRequestEditorFn(provider.Intercept))
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := client.GetSmokeIpWithResponse(ctx, ip)
+	if err != nil {
+		return err
+	}
+
+	switch {
+	case resp.JSON404 != nil:
+		return errors.New("ip not found")
+	case resp.JSON403 != nil:
+		return errors.New("forbidden")
+	case resp.JSON500 != nil:
+		return errors.New("internal server error")
+	case resp.JSON429 != nil:
+		return errors.New("too many requests")
+	case resp.JSON400 != nil:
+		return errors.New("bad request")
+	case resp.JSON200 == nil:
+		return fmt.Errorf("unexpected error %d", resp.StatusCode())
+	}
+
+	ctiObj := resp.JSON200
+
+	var out []byte
+
+	// re-encode (todo: yaml, human)
+
+	out, err = json.MarshalIndent(ctiObj, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	fmt.Println(string(out))
+
+	return nil
+}
+
+func (cli *cliSmokeIP) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "smoke-ip",
+		Short: "Query the smoke data with a given IP",
+		Args:  cobra.ExactArgs(1),
+		RunE:  func(cmd *cobra.Command, args []string) error {
+			return cli.smokeip(args[0])
+		},
+	}
+
+	return cmd
+}

go.mod

@@ -30,6 +30,7 @@ require (
 	github.com/crowdsecurity/grokky v0.2.1
 	github.com/crowdsecurity/machineid v1.0.2
 	github.com/davecgh/go-spew v1.1.1
+	github.com/deepmap/oapi-codegen v1.16.2
 	github.com/dghubble/sling v1.3.0
 	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-connections v0.4.0
@@ -46,7 +47,7 @@ require (
 	github.com/gofrs/uuid v4.0.0+incompatible
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/google/go-querystring v1.0.0
-	github.com/google/uuid v1.3.0
+	github.com/google/uuid v1.5.0
 	github.com/google/winops v0.0.0-20230712152054-af9b550d0601
 	github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e
 	github.com/gorilla/websocket v1.5.0
@@ -59,10 +60,11 @@ require (
 	github.com/jarcoal/httpmock v1.1.0
 	github.com/jszwec/csvutil v1.5.1
 	github.com/lithammer/dedent v1.1.0
-	github.com/mattn/go-isatty v0.0.19
+	github.com/mattn/go-isatty v0.0.20
 	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
 	github.com/nxadm/tail v1.4.8
+	github.com/oapi-codegen/runtime v1.1.1
 	github.com/oschwald/geoip2-golang v1.4.0
 	github.com/oschwald/maxminddb-golang v1.8.0
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
@@ -82,7 +84,7 @@ require (
 	github.com/wasilibs/go-re2 v1.3.0
 	github.com/xhit/go-simple-mail/v2 v2.16.0
 	golang.org/x/crypto v0.17.0
-	golang.org/x/mod v0.11.0
+	golang.org/x/mod v0.12.0
 	golang.org/x/sys v0.15.0
 	golang.org/x/text v0.14.0
 	google.golang.org/grpc v1.56.3
@@ -99,11 +101,13 @@ require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 // indirect
+	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bytedance/sonic v1.9.1 // indirect
-	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
+	github.com/bytedance/sonic v1.10.0-rc3 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
+	github.com/chenzhuoyu/iasm v0.9.0 // indirect
 	github.com/corazawaf/libinjection-go v0.1.2 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
@@ -123,7 +127,7 @@ require (
 	github.com/go-openapi/spec v0.20.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.14.0 // indirect
+	github.com/go-playground/validator/v10 v10.14.1 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -148,13 +152,13 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.17.3 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magefile/mage v1.15.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
@@ -169,7 +173,7 @@ require (
 	github.com/oklog/run v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.9 // indirect
 	github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -197,13 +201,13 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	github.com/zclconf/go-cty v1.8.0 // indirect
 	go.mongodb.org/mongo-driver v1.9.4 // indirect
-	golang.org/x/arch v0.3.0 // indirect
+	golang.org/x/arch v0.4.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/term v0.15.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.12.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect