INSIGHTS-328 - OPA supporting external libraries (#207)

* adds push external-opa command

* add enable/disable handling

* add tests and json schema

* fix copyright

pkg/cli/push_external_opa.go

@@ -0,0 +1,56 @@
+// Copyright 2024 FairwindsOps Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cli
+
+import (
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/fairwindsops/insights-cli/pkg/opa"
+)
+
+var pushExternalOPAFile string
+var pushExternalOPASubDir string
+var pushExternalOPAHeaders []string
+
+const defaultPushExternalOPASubDir = "external-opa"
+
+func init() {
+	pushExternalOPACmd.PersistentFlags().BoolVarP(&deleteMissingOPA, "delete", "D", false, "Delete any OPA policies from Insights that are not present in the external OPA file definition.")
+	// This flag sets a variable defined in the parent `push` command.
+	pushExternalOPACmd.PersistentFlags().StringVarP(&pushExternalOPASubDir, "subdirectory", "s", defaultPushExternalOPASubDir, "Sub-directory within push-directory, to contain the external OPA file definition.")
+	pushExternalOPACmd.PersistentFlags().StringVarP(&pushExternalOPAFile, "file", "f", "external-sources.yaml", "file name of the external OPA file definition.")
+	pushExternalOPACmd.PersistentFlags().StringSliceVarP(&pushExternalOPAHeaders, "header", "", []string{}, "these headers are passed to the external service provider. i.e.: for authentication")
+	pushCmd.AddCommand(pushExternalOPACmd)
+}
+
+var pushExternalOPACmd = &cobra.Command{
+	Use:    "external-opa",
+	Short:  "Push External OPA policies.",
+	Long:   "Push External OPA policies to Insights.",
+	PreRun: validateAndLoadInsightsAPIConfigWrapper,
+	Run: func(cmd *cobra.Command, args []string) {
+		org := configurationObject.Options.Organization
+		host := configurationObject.Options.Hostname
+		filePath := fmt.Sprintf("%s/%s/%s", pushDir, pushExternalOPASubDir, pushExternalOPAFile)
+		err := opa.PushExternalOPAChecks(filePath, org, insightsToken, pushExternalOPAHeaders, host, deleteMissingOPA, pushDryRun)
+		if err != nil {
+			logrus.Fatalf("Unable to push external OPA checks: %v", err)
+		}
+		logrus.Infoln("Push succeeded.")
+	},
+}

pkg/cli/root.go

@@ -64,7 +64,7 @@ func (c configuration) CheckForErrors() error {
 }
 
 // RUn executes the cobra root command, and returns an exit value depending on
-// whether an error occured.
+// whether an error occurred.
 func Run() (exitValue int) {
 	if err := rootCmd.Execute(); err != nil {
 		logrus.Error(err)

pkg/directory/scan.go

@@ -104,7 +104,7 @@ func findRegoFilesOtherThanPolicy(dir string) ([]string, error) {
 }
 
 // fileNamesAreUnique returns true when no file names repeat in the given
-// list. For exampole, given input ["dir1/file.txt", "dir2/file.txt"] would
+// list. For example, given input ["dir1/file.txt", "dir2/file.txt"] would
 // return false, and duplicateNames of map["file.txt"]{"dir1", "dir2"}.
 func fileNamesAreUnique(files []string) (alreadyUnique bool, duplicateNames map[string][]string) {
 	duplicateNames = make(map[string][]string)

pkg/models/model.go

@@ -30,11 +30,13 @@ type OutputModel struct {
 
 // CustomCheckModel is a model for the API endpoint to receive a Custom Check for OPA
 type CustomCheckModel struct {
-	CheckName string `json:"-" yaml:"-"`
-	Version   float32
-	Output    OutputModel
-	Rego      string
-	Instances []CustomCheckInstanceModel `json:"-" yaml:"-"`
+	CheckName   string `json:"-" yaml:"-"`
+	Version     float32
+	Output      OutputModel
+	Rego        string
+	Instances   []CustomCheckInstanceModel `json:"-" yaml:"-"`
+	Description string
+	Disabled    *bool
 }
 
 // CustomCheckInstanceModel is a model for the API endpoint to receive an Instance for a Custom Check in OPA

pkg/opa/opa.go

@@ -28,7 +28,6 @@ import (
 	"github.com/xlab/treeprint"
 	"gopkg.in/yaml.v3"
 
-	"github.com/fairwindsops/insights-cli/pkg/directory"
 	"github.com/fairwindsops/insights-cli/pkg/models"
 )
 
@@ -43,19 +42,8 @@ type CompareResults struct {
 }
 
 // CompareChecks compares a folder vs the checks returned by the API.
-func CompareChecks(folder, org, token, hostName string, deleteMissing bool) (CompareResults, error) {
+func CompareChecks(folder, org, token, hostName string, fileChecks []models.CustomCheckModel, deleteMissing bool) (CompareResults, error) {
 	var results CompareResults
-	files, err := directory.ScanOPAFolder(folder)
-	if err != nil {
-		logrus.Error("Error scanning directory")
-		return results, err
-	}
-
-	fileChecks, err := getChecksFromFiles(files)
-	if err != nil {
-		logrus.Error("Error Reading checks from files")
-		return results, err
-	}
 	apiChecks, err := GetChecks(org, token, hostName)
 	if err != nil {
 		logrus.Error("Error getting checks from Insights")

pkg/opa/opa_calls.go

@@ -17,15 +17,19 @@ package opa
 import (
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
+	"strings"
 
+	"github.com/fairwindsops/insights-cli/pkg/directory"
 	"github.com/fairwindsops/insights-cli/pkg/models"
 	"github.com/fairwindsops/insights-cli/pkg/utils"
 	"github.com/fairwindsops/insights-cli/pkg/version"
 	opaPlugin "github.com/fairwindsops/insights-plugins/plugins/opa/pkg/opa"
 	"github.com/imroc/req"
 	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
 )
 
 const opaURLFormat = "%s/v0/organizations/%s/opa/customChecks"
@@ -89,10 +93,15 @@ func DeleteCheck(check models.CustomCheckModel, org, token, hostName string) err
 	return nil
 }
 
+type PutCheckRequest struct {
+	Rego, Description string
+	Disabled          *bool
+}
+
 // PutCheck upserts an OPA Check to Fairwinds Insights
 func PutCheck(check models.CustomCheckModel, org, token, hostName string) error {
 	url := fmt.Sprintf(opaPutCheckURLFormat, hostName, org, check.CheckName, check.Version)
-	resp, err := req.Put(url, utils.GetHeaders(version.GetVersion(), token), req.BodyJSON(&check))
+	resp, err := req.Put(url, utils.GetHeaders(version.GetVersion(), token), req.BodyJSON(PutCheckRequest{Rego: check.Rego, Description: check.Description, Disabled: check.Disabled}))
 	if err != nil {
 		return err
 	}
@@ -132,19 +141,27 @@ func PutInstance(instance models.CustomCheckInstanceModel, org, token, hostName
 }
 
 // PushOPAChecks pushes OPA checks to Insights.
-func PushOPAChecks(pushDir, org, insightsToken, host string, deleteMissing, dryrun bool) error {
+func PushOPAChecks(pushDir, org, insightsToken, host string, deleteMissing, dryRun bool) error {
 	logrus.Debugln("Pushing OPA policies")
 	_, err := os.Stat(pushDir)
 	if err != nil {
 		return err
 	}
-	results, err := CompareChecks(pushDir, org, insightsToken, host, deleteMissing)
+	files, err := directory.ScanOPAFolder(pushDir)
+	if err != nil {
+		return fmt.Errorf("error scanning directory: %w", err)
+	}
+	fileChecks, err := getChecksFromFiles(files)
+	if err != nil {
+		return fmt.Errorf("error Reading checks from files: %w", err)
+	}
+	results, err := CompareChecks(pushDir, org, insightsToken, host, fileChecks, deleteMissing)
 	if err != nil {
 		return err
 	}
 	for _, instance := range results.InstanceDelete {
 		logrus.Infof("Deleting instance: %s for OPA policy %s", instance.InstanceName, instance.CheckName)
-		if !dryrun {
+		if !dryRun {
 			err := DeleteInstance(instance, org, insightsToken, host)
 			if err != nil {
 				return err
@@ -153,7 +170,7 @@ func PushOPAChecks(pushDir, org, insightsToken, host string, deleteMissing, dryr
 	}
 	for _, check := range results.CheckDelete {
 		logrus.Infof("Deleting OPA policy: %s", check.CheckName)
-		if !dryrun {
+		if !dryRun {
 			err := DeleteCheck(check, org, insightsToken, host)
 			if err != nil {
 				return err
@@ -162,7 +179,7 @@ func PushOPAChecks(pushDir, org, insightsToken, host string, deleteMissing, dryr
 	}
 	for _, check := range results.CheckInsert {
 		logrus.Infof("Adding v%.0f OPA policy: %s", check.Version, check.CheckName)
-		if !dryrun {
+		if !dryRun {
 			err := PutCheck(check, org, insightsToken, host)
 			if err != nil {
 				return err
@@ -171,7 +188,7 @@ func PushOPAChecks(pushDir, org, insightsToken, host string, deleteMissing, dryr
 	}
 	for _, check := range results.CheckUpdate {
 		logrus.Infof("Updating v%.0f OPA policy: %s", check.Version, check.CheckName)
-		if !dryrun {
+		if !dryRun {
 			err := PutCheck(check, org, insightsToken, host)
 			if err != nil {
 				return err
@@ -180,7 +197,7 @@ func PushOPAChecks(pushDir, org, insightsToken, host string, deleteMissing, dryr
 	}
 	for _, instance := range results.InstanceInsert {
 		logrus.Infof("Adding instance: %s for OPA policy %s", instance.InstanceName, instance.CheckName)
-		if !dryrun {
+		if !dryRun {
 			err := PutInstance(instance, org, insightsToken, host)
 			if err != nil {
 				return err
@@ -189,7 +206,7 @@ func PushOPAChecks(pushDir, org, insightsToken, host string, deleteMissing, dryr
 	}
 	for _, instance := range results.InstanceUpdate {
 		logrus.Infof("Updating instance: %s for OPA policy %s", instance.InstanceName, instance.CheckName)
-		if !dryrun {
+		if !dryRun {
 			err := PutInstance(instance, org, insightsToken, host)
 			if err != nil {
 				return err
@@ -199,3 +216,154 @@ func PushOPAChecks(pushDir, org, insightsToken, host string, deleteMissing, dryr
 	logrus.Debugln("Done pushing OPA policies")
 	return nil
 }
+
+// PushExternalOPAChecks pushes external OPA checks to Insights.
+func PushExternalOPAChecks(filePath, org, insightsToken string, headers []string, host string, deleteMissing, dryRun bool) error {
+	logrus.Debugln("Pushing external OPA policies")
+	_, err := os.Stat(filePath)
+	if err != nil {
+		return fmt.Errorf("error opening file: %w", err)
+	}
+
+	f, err := os.Open(filePath)
+	if err != nil {
+		return fmt.Errorf("error opening file: %w", err)
+	}
+	defer f.Close()
+
+	b, err := io.ReadAll(f)
+	if err != nil {
+		return fmt.Errorf("error reading file: %w", err)
+	}
+
+	checks, err := getExternalChecksFromFile(b, headers)
+	if err != nil {
+		return fmt.Errorf("error getting remote checks: %w", err)
+	}
+
+	results, err := CompareChecks(filePath, org, insightsToken, host, checks, deleteMissing)
+	if err != nil {
+		return fmt.Errorf("error comparing checks: %w", err)
+	}
+	for _, instance := range results.InstanceDelete {
+		logrus.Infof("Deleting instance: %s for OPA policy %s", instance.InstanceName, instance.CheckName)
+		if !dryRun {
+			err := DeleteInstance(instance, org, insightsToken, host)
+			if err != nil {
+				return fmt.Errorf("error deleting instance: %w", err)
+			}
+		}
+	}
+	for _, check := range results.CheckDelete {
+		logrus.Infof("Deleting OPA policy: %s", check.CheckName)
+		if !dryRun {
+			err := DeleteCheck(check, org, insightsToken, host)
+			if err != nil {
+				return fmt.Errorf("error deleting check: %w", err)
+			}
+		}
+	}
+	for _, check := range results.CheckInsert {
+		logrus.Infof("Adding v%.0f OPA policy: %s", check.Version, check.CheckName)
+		if !dryRun {
+			err := PutCheck(check, org, insightsToken, host)
+			if err != nil {
+				return fmt.Errorf("error adding check: %w", err)
+			}
+		}
+	}
+	for _, check := range results.CheckUpdate {
+		logrus.Infof("Updating v%.0f OPA policy: %s", check.Version, check.CheckName)
+		if !dryRun {
+			err := PutCheck(check, org, insightsToken, host)
+			if err != nil {
+				return fmt.Errorf("error updating check: %w", err)
+			}
+		}
+	}
+	for _, instance := range results.InstanceInsert {
+		logrus.Infof("Adding instance: %s for OPA policy %s", instance.InstanceName, instance.CheckName)
+		if !dryRun {
+			err := PutInstance(instance, org, insightsToken, host)
+			if err != nil {
+				return fmt.Errorf("error adding instance: %w", err)
+			}
+		}
+	}
+	for _, instance := range results.InstanceUpdate {
+		logrus.Infof("Updating instance: %s for OPA policy %s", instance.InstanceName, instance.CheckName)
+		if !dryRun {
+			err := PutInstance(instance, org, insightsToken, host)
+			if err != nil {
+				return fmt.Errorf("error updating instance: %w", err)
+			}
+		}
+	}
+	logrus.Debugln("Done pushing external OPA policies")
+	return nil
+}
+
+type externalSource struct {
+	ExternalSources []externalSourceItem `yaml:"externalSources"`
+}
+
+type externalSourceItem struct {
+	Name        string `yaml:"name"`
+	Description string `yaml:"description"`
+	URL         string `yaml:"url"`
+	Enabled     *bool  `yaml:"enabled"`
+}
+
+// getExternalChecksFromFile reads the external sources file and fetches the OPA checks from them
+func getExternalChecksFromFile(fileContent []byte, headers []string) ([]models.CustomCheckModel, error) {
+	var externalSources externalSource
+	err := yaml.Unmarshal(fileContent, &externalSources)
+	if err != nil {
+		return nil, fmt.Errorf("error unmarshalling yaml: %w", err)
+	}
+
+	if len(externalSources.ExternalSources) == 0 {
+		return []models.CustomCheckModel{}, nil
+	}
+
+	var checks []models.CustomCheckModel
+	for _, source := range externalSources.ExternalSources {
+		logrus.Debugf("getting checks from %s", source.URL)
+		resp, err := req.Get(source.URL, req.Header(formatHeaders(headers)))
+		if err != nil {
+			return nil, fmt.Errorf("error getting remote checks: %w", err)
+		}
+		if resp.Response().StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("error getting remote checks: invalid response code (%v, expected 200)", resp.Response().StatusCode)
+		}
+		rego, err := resp.ToString()
+		if err != nil {
+			return nil, fmt.Errorf("error unmarshalling remote checks: %w", err)
+		}
+		checks = append(checks, models.CustomCheckModel{
+			CheckName:   source.Name,
+			Description: source.Description,
+			Rego:        rego,
+			Version:     2.0,
+			Disabled:    utils.InvertBoolPointer(source.Enabled),
+		})
+	}
+	return checks, nil
+}
+
+func formatHeaders(headers []string) map[string]string {
+	r := map[string]string{}
+	for _, s := range headers {
+		parts := strings.Split(s, ":")
+		if len(parts) != 2 {
+			logrus.Warnf("invalid header: '%s' - should be formatted as 'key:value'", s)
+			continue
+		}
+		key := strings.TrimSpace(parts[0])
+		value := strings.TrimSpace(parts[1])
+		r[key] = value
+	}
+
+	logrus.Debugf("headers: %v", r)
+	return r
+}