diff --git a/test/engine/setup/controller/kubernetes.go b/test/engine/setup/controller/kubernetes.go index 99765c85a4..2cb7ec4d53 100644 --- a/test/engine/setup/controller/kubernetes.go +++ b/test/engine/setup/controller/kubernetes.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "os" + "path/filepath" "time" "github.com/avast/retry-go/v4" @@ -25,9 +26,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/restmapper" "github.com/alibaba/ilogtail/test/config" ) @@ -50,7 +52,8 @@ type DaemonSetController struct { } type DynamicController struct { - dynamicClient dynamic.Interface + discoveryClient discovery.DiscoveryInterface + dynamicClient dynamic.Interface } func NewDeploymentController(k8sClient *kubernetes.Clientset) *DeploymentController { @@ -215,8 +218,8 @@ func (c *DaemonSetController) GetDaemonSetPods(dsName, dsNamespace string) (*cor return pods, nil } -func NewDynamicController(dynamicClient dynamic.Interface) *DynamicController { - return &DynamicController{dynamicClient: dynamicClient} +func NewDynamicController(dynamicClient dynamic.Interface, discoveryClient discovery.DiscoveryInterface) *DynamicController { + return &DynamicController{dynamicClient: dynamicClient, discoveryClient: discoveryClient} } func (c *DynamicController) Apply(filePath string) error { @@ -267,7 +270,8 @@ func (c *DynamicController) Delete(filePath string) error { func (c *DynamicController) parseObjFromYaml(filePath string) (*meta.RESTMapping, *unstructured.Unstructured, error) { // Read the YAML file - yamlFile, err := os.ReadFile(filePath) + basePath := "test_cases" + yamlFile, err := os.ReadFile(filepath.Join(basePath, filePath)) if err != nil { return nil, nil, err } @@ -280,8 +284,11 @@ func (c *DynamicController) parseObjFromYaml(filePath string) (*meta.RESTMapping return nil, nil, err } - // Retrieve the REST mapping for the GVK - restMapper := meta.NewDefaultRESTMapper(scheme.Scheme.PreferredVersionAllGroups()) + apiGroupResources, err := restmapper.GetAPIGroupResources(c.discoveryClient) + if err != nil { + return nil, nil, err + } + restMapper := restmapper.NewDiscoveryRESTMapper(apiGroupResources) mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) if err != nil { return nil, nil, err diff --git a/test/engine/setup/k8s.go b/test/engine/setup/k8s.go index 85eaef8e6d..826b1e3644 100644 --- a/test/engine/setup/k8s.go +++ b/test/engine/setup/k8s.go @@ -19,6 +19,7 @@ import ( "crypto/rand" "fmt" "math/big" + "sort" "strings" corev1 "k8s.io/api/core/v1" @@ -80,12 +81,15 @@ func (k *K8sEnv) ExecOnLogtail(command string) (string, error) { } } results := make([]string, 0) + sort.Slice(pods.Items, func(i, j int) bool { + return pods.Items[i].Name < pods.Items[j].Name + }) for _, pod := range pods.Items { - if result, err := k.execInPod(k.config, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name, []string{"bash", "-c", command}); err != nil { + result, err := k.execInPod(k.config, pod.Namespace, pod.Name, pod.Spec.Containers[0].Name, []string{"bash", "-c", command}) + if err != nil { return "", err - } else { - results = append(results, result) } + results = append(results, result) } return strings.Join(results, "\n"), nil } @@ -152,7 +156,10 @@ func (k *K8sEnv) init() { k.daemonsetController = controller.NewDaemonSetController(k.k8sClient) dynamicClient, err := dynamic.NewForConfig(c) - k.dynamicController = controller.NewDynamicController(dynamicClient) + if err != nil { + panic(err) + } + k.dynamicController = controller.NewDynamicController(dynamicClient, k.k8sClient.Discovery()) } func (k *K8sEnv) execInPod(config *rest.Config, namespace, podName, containerName string, command []string) (string, error) { diff --git a/test/engine/steps.go b/test/engine/steps.go index 43df1f980a..16fd0c622a 100644 --- a/test/engine/steps.go +++ b/test/engine/steps.go @@ -48,10 +48,8 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.When(`^generate \{(\d+)\} apsara logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.Apsara) ctx.When(`^generate \{(\d+)\} delimiter logs to file \{(.*)\}, with interval \{(\d+)\}ms, with delimiter \{(.*)\} and quote \{(.*)\}$`, trigger.DelimiterSingle) ctx.When(`^generate \{(\d+)\} multiline delimiter logs to file \{(.*)\}, with interval \{(\d+)\}ms, with delimiter \{(.*)\} and quote \{(.*)\}$`, trigger.DelimiterMultiline) - ctx.When(`^generate \{(\d+)\} json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.JsonSingle) - ctx.When(`^generate \{(\d+)\} multiline json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.JsonMultiline) - ctx.When(`^generate \{(\d+)\} logs to stdout, with interval \{(\d+)\}ms$`, trigger.Stdout) - ctx.When(`^generate \{(\d+)\} logs to stderr, with interval \{(\d+)\}ms$`, trigger.Stderr) + ctx.When(`^generate \{(\d+)\} json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.JSONSingle) + ctx.When(`^generate \{(\d+)\} multiline json logs to file \{(.*)\}, with interval \{(\d+)\}ms$`, trigger.JSONMultiline) ctx.When(`^execute \{(\d+)\} commands to generate file security events on files \{(.*)\}$`, trigger.TrigerFileSecurityEvents) // ------------------------------------------ diff --git a/test/engine/trigger/generator/helper.go b/test/engine/trigger/generator/helper.go index 9e58df2753..2514bb2614 100644 --- a/test/engine/trigger/generator/helper.go +++ b/test/engine/trigger/generator/helper.go @@ -14,8 +14,9 @@ package generator import ( + "crypto/rand" "fmt" - "math/rand" + "math/big" "os" "strconv" "text/template" @@ -66,12 +67,14 @@ func string2Template(strings []string) []*template.Template { } func getRandomLogLevel() string { - return Levels[rand.Intn(len(Levels))] + randInt, _ := rand.Int(rand.Reader, big.NewInt(int64(len(Levels)))) + return Levels[randInt.Int64()] } func getRandomMark() string { marks := []string{"-", "F"} - return marks[rand.Intn(2)] + randInt, _ := rand.Int(rand.Reader, big.NewInt(int64(len(marks)))) + return marks[randInt.Int64()] } func getEnvOrDefault(env, fallback string) string { diff --git a/test/engine/trigger/generator/json_test.go b/test/engine/trigger/generator/json_test.go index 54b98ec3d6..7f17f0d438 100644 --- a/test/engine/trigger/generator/json_test.go +++ b/test/engine/trigger/generator/json_test.go @@ -22,8 +22,8 @@ import ( "time" ) -// TestGenerateJsonSingle will be executed in the environment being collected. -func TestGenerateJsonSingle(t *testing.T) { +// TestGenerateJSONSingle will be executed in the environment being collected. +func TestGenerateJSONSingle(t *testing.T) { config, err := getGenerateFileLogConfigFromEnv() if err != nil { t.Fatalf("get generate file log config from env failed: %v", err) @@ -56,7 +56,7 @@ func TestGenerateJsonSingle(t *testing.T) { } else { currentTime = strconv.FormatInt(time.Now().UnixNano()/1000, 10) } - err = testLogConentTmpl[logIndex].Execute(file, map[string]interface{}{ + testLogConentTmpl[logIndex].Execute(file, map[string]interface{}{ "Mark": getRandomMark(), "FileNo": fileNo, "LogNo": logNo + i, @@ -71,7 +71,7 @@ func TestGenerateJsonSingle(t *testing.T) { } } -func TestGenerateJsonMultiline(t *testing.T) { +func TestGenerateJSONMultiline(t *testing.T) { config, err := getGenerateFileLogConfigFromEnv() if err != nil { t.Fatalf("get generate file log config from env failed: %v", err) @@ -103,7 +103,7 @@ func TestGenerateJsonMultiline(t *testing.T) { fileNo := rand.Intn(10000) for i := 0; i < config.TotalLog; i++ { currentTime := time.Now().Format("2006-01-02T15:04:05.999999999") - err = testLogConentTmpl[logIndex].Execute(file, map[string]interface{}{ + testLogConentTmpl[logIndex].Execute(file, map[string]interface{}{ "Mark": getRandomMark(), "FileNo": fileNo, "LogNo": logNo + i, diff --git a/test/engine/trigger/generator/stdout_test.go b/test/engine/trigger/generator/stdout_test.go deleted file mode 100644 index f62cfe021a..0000000000 --- a/test/engine/trigger/generator/stdout_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2024 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package generator - -import ( - "io" - "math/rand" - "os" - "testing" - "time" -) - -// TestGenerateStdout will be executed in the environment being collected. -func TestGenerateStdout(t *testing.T) { - config, err := getGenerateFileLogConfigFromEnv("TestMode") - if err != nil { - t.Fatalf("get config failed: %v", err) - return - } - testLogContentTmpl := string2Template([]string{ - `{{.LogNo}} [{{.Time}}] [INFO] "这是一条消息，password:123456"`, - `{{.LogNo}} [{{.Time}}] [WARN] "这是一条消息，password:123456，这是第二条消息，password:00000"`, - `{{.LogNo}} [{{.Time}}] [ERROR] "这是一条消息"`, - `{{.LogNo}} [{{.Time}}] [DEBUG] "这是一条消息，password:123456"`, - }) - - logIndex := 0 - logNo := rand.Intn(10000) - var wr io.Writer - if config.Custom["TestMode"] == "stdout" { - wr = os.Stdout - } else { - wr = os.Stderr - } - for i := 0; i < config.TotalLog; i++ { - err = testLogContentTmpl[logIndex].Execute(wr, map[string]interface{}{ - "Time": time.Now().Format("2006-01-02T15:04:05.000000"), - "LogNo": logNo + i, - }) - if err != nil { - t.Fatalf("write log failed: %v", err) - return - } - time.Sleep(time.Duration(config.Interval * int(time.Millisecond))) - logIndex++ - if logIndex >= len(testLogContentTmpl) { - logIndex = 0 - } - } -} diff --git a/test/engine/trigger/trigger.go b/test/engine/trigger/trigger.go index 2de29b4618..63653c5cb6 100644 --- a/test/engine/trigger/trigger.go +++ b/test/engine/trigger/trigger.go @@ -37,12 +37,12 @@ func RegexMultiline(ctx context.Context, totalLog int, path string, interval int return generate(ctx, totalLog, path, interval, "TestGenerateRegexLogMultiline") } -func JsonSingle(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateJsonSingle") +func JSONSingle(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, totalLog, path, interval, "TestGenerateJSONSingle") } -func JsonMultiline(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { - return generate(ctx, totalLog, path, interval, "TestGenerateJsonMultiline") +func JSONMultiline(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { + return generate(ctx, totalLog, path, interval, "TestGenerateJSONMultiline") } func Apsara(ctx context.Context, totalLog int, path string, interval int) (context.Context, error) { @@ -57,50 +57,6 @@ func DelimiterMultiline(ctx context.Context, totalLog int, path string, interval return generate(ctx, totalLog, path, interval, "TestGenerateDelimiterMultiline", "Delimiter", delimiter, "Quote", quote) } -func Stdout(ctx context.Context, totalLog int, interval int) (context.Context, error) { - time.Sleep(3 * time.Second) - command := getRunTriggerCommand("TestGenerateStdout") - var triggerCommand strings.Builder - template := template.Must(template.New("trigger").Parse(triggerTemplate)) - if err := template.Execute(&triggerCommand, map[string]interface{}{ - "WorkDir": "", - "TotalLog": totalLog, - "Interval": interval, - "TestMode": "stdout", - "Filename": "", - "Custom": "", - "Command": command, - }); err != nil { - return ctx, err - } - if _, err := setup.Env.ExecOnSource(ctx, triggerCommand.String()); err != nil { - return ctx, err - } - return ctx, nil -} - -func Stderr(ctx context.Context, totalLog int, interval int) (context.Context, error) { - time.Sleep(3 * time.Second) - command := getRunTriggerCommand("TestGenerateStdout") - var triggerCommand strings.Builder - template := template.Must(template.New("trigger").Parse(triggerTemplate)) - if err := template.Execute(&triggerCommand, map[string]interface{}{ - "WorkDir": "", - "TotalLog": totalLog, - "Interval": interval, - "TestMode": "stderr", - "Filename": "", - "Custom": "", - "Command": command, - }); err != nil { - return ctx, err - } - if _, err := setup.Env.ExecOnSource(ctx, triggerCommand.String()); err != nil { - return ctx, err - } - return ctx, nil -} - func generate(ctx context.Context, totalLog int, path string, interval int, commandName string, customKV ...string) (context.Context, error) { time.Sleep(3 * time.Second) command := getRunTriggerCommand(commandName)