From 1607c86ae3a619dc7ed7a0c0c77578d8c3468ac6 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 18 Jul 2023 22:01:13 +0200 Subject: [PATCH 1/2] Integration tests: fixed couldn't stat file race condition --- pkg/testing/runner/archiver.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/pkg/testing/runner/archiver.go b/pkg/testing/runner/archiver.go index 616508f0acb..ee57c7ab0aa 100644 --- a/pkg/testing/runner/archiver.go +++ b/pkg/testing/runner/archiver.go @@ -24,21 +24,32 @@ func createRepoZipArchive(ctx context.Context, dir string, dest string) error { if err != nil { return fmt.Errorf("failed to get absolute path to %s: %w", dir, err) } - var stdout bytes.Buffer - processHandler, err := process.Start("git", process.WithContext(ctx), process.WithArgs([]string{"ls-files", "-z"}), process.WithCmdOptions(attachOut(&stdout), workDir(dir))) + + var allOutput bytes.Buffer + var trackedOutput bytes.Buffer + processHandler, err := process.Start("git", process.WithContext(ctx), process.WithArgs([]string{"ls-files", "-z"}), process.WithCmdOptions(attachOut(&trackedOutput), workDir(dir))) if err != nil { return fmt.Errorf("failed to run git ls-files: %w", err) } + _, err = io.Copy(&allOutput, &trackedOutput) + if err != nil { + return fmt.Errorf("failed to read stdout of git ls-files: %w", err) + } processDone := <-processHandler.Wait() if processDone.ExitCode() != 0 { return fmt.Errorf("failed to run git ls-files: exited code %d", processDone.ExitCode()) } // Add files that are not yet tracked in git. Prevents a footcannon where someone writes code to a new file, then tests it before they add to git - processHandler, err = process.Start("git", process.WithContext(ctx), process.WithArgs([]string{"ls-files", "--exclude-standard", "-o", "-z"}), process.WithCmdOptions(attachOut(&stdout), workDir(dir))) + var untrackedOutput bytes.Buffer + processHandler, err = process.Start("git", process.WithContext(ctx), process.WithArgs([]string{"ls-files", "--exclude-standard", "-o", "-z"}), process.WithCmdOptions(attachOut(&untrackedOutput), workDir(dir))) if err != nil { return fmt.Errorf("failed to run git ls-files -o: %w", err) } + _, err = io.Copy(&allOutput, &untrackedOutput) + if err != nil { + return fmt.Errorf("failed to read stdout of git ls-files -o: %w", err) + } processDone = <-processHandler.Wait() if processDone.ExitCode() != 0 { return fmt.Errorf("failed to run git ls-files -o: exited code %d", processDone.ExitCode()) @@ -53,7 +64,7 @@ func createRepoZipArchive(ctx context.Context, dir string, dest string) error { zw := zip.NewWriter(archive) defer zw.Close() - s := bufio.NewScanner(&stdout) + s := bufio.NewScanner(&allOutput) s.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) { if i := strings.IndexRune(string(data), '\x00'); i >= 0 { return i + 1, data[0:i], nil From 95cb9567b67fd547e755ba65284f9ed6438d7017 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 18 Jul 2023 23:40:25 +0200 Subject: [PATCH 2/2] Use cmd instead of async process --- pkg/testing/runner/archiver.go | 42 ++++++++++++---------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/pkg/testing/runner/archiver.go b/pkg/testing/runner/archiver.go index ee57c7ab0aa..7b4fb759217 100644 --- a/pkg/testing/runner/archiver.go +++ b/pkg/testing/runner/archiver.go @@ -15,8 +15,6 @@ import ( "os/exec" "path/filepath" "strings" - - "github.com/elastic/elastic-agent/pkg/core/process" ) func createRepoZipArchive(ctx context.Context, dir string, dest string) error { @@ -25,35 +23,21 @@ func createRepoZipArchive(ctx context.Context, dir string, dest string) error { return fmt.Errorf("failed to get absolute path to %s: %w", dir, err) } - var allOutput bytes.Buffer - var trackedOutput bytes.Buffer - processHandler, err := process.Start("git", process.WithContext(ctx), process.WithArgs([]string{"ls-files", "-z"}), process.WithCmdOptions(attachOut(&trackedOutput), workDir(dir))) - if err != nil { - return fmt.Errorf("failed to run git ls-files: %w", err) - } - _, err = io.Copy(&allOutput, &trackedOutput) + projectFilesOutput, err := cmdBufferedOutput(exec.Command("git", "ls-files", "-z"), dir) if err != nil { - return fmt.Errorf("failed to read stdout of git ls-files: %w", err) - } - processDone := <-processHandler.Wait() - if processDone.ExitCode() != 0 { - return fmt.Errorf("failed to run git ls-files: exited code %d", processDone.ExitCode()) + return err } // Add files that are not yet tracked in git. Prevents a footcannon where someone writes code to a new file, then tests it before they add to git - var untrackedOutput bytes.Buffer - processHandler, err = process.Start("git", process.WithContext(ctx), process.WithArgs([]string{"ls-files", "--exclude-standard", "-o", "-z"}), process.WithCmdOptions(attachOut(&untrackedOutput), workDir(dir))) + untrackedOutput, err := cmdBufferedOutput(exec.Command("git", "ls-files", "--exclude-standard", "-o", "-z"), dir) if err != nil { - return fmt.Errorf("failed to run git ls-files -o: %w", err) + return err } - _, err = io.Copy(&allOutput, &untrackedOutput) + + _, err = io.Copy(&projectFilesOutput, &untrackedOutput) if err != nil { return fmt.Errorf("failed to read stdout of git ls-files -o: %w", err) } - processDone = <-processHandler.Wait() - if processDone.ExitCode() != 0 { - return fmt.Errorf("failed to run git ls-files -o: exited code %d", processDone.ExitCode()) - } archive, err := os.Create(dest) if err != nil { @@ -64,7 +48,7 @@ func createRepoZipArchive(ctx context.Context, dir string, dest string) error { zw := zip.NewWriter(archive) defer zw.Close() - s := bufio.NewScanner(&allOutput) + s := bufio.NewScanner(&projectFilesOutput) s.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) { if i := strings.IndexRune(string(data), '\x00'); i >= 0 { return i + 1, data[0:i], nil @@ -116,9 +100,13 @@ func createRepoZipArchive(ctx context.Context, dir string, dest string) error { return nil } -func workDir(dir string) process.CmdOption { - return func(cmd *exec.Cmd) error { - cmd.Dir = dir - return nil +func cmdBufferedOutput(cmd *exec.Cmd, workDir string) (bytes.Buffer, error) { + var stdoutBuf bytes.Buffer + cmd.Dir = workDir + cmd.Stdout = &stdoutBuf + err := cmd.Run() + if err != nil { + return *bytes.NewBufferString(""), fmt.Errorf("failed to run cmd %s: %w", strings.Join(cmd.Args, " "), err) } + return stdoutBuf, nil }