From a1e8891c20b7484255ddfb135ad802f2275edc6f Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Mon, 23 Oct 2023 10:34:40 -0400 Subject: [PATCH 1/2] trt-1322: update ok checks and output last body on failure --- README.md | 2 +- cmd/main.go | 107 +++++++++++++++++++++++++++++++--------------------- go.mod | 10 ++++- go.sum | 9 +++++ 4 files changed, 81 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 94f50eb..97b4aed 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ $ ./gangway-cli \ --initial "registry.ci.openshift.org/ocp/release:4.14.0-0.nightly-2023-07-05-071214" \ --latest "registry.ci.openshift.org/ocp/release:4.14.0-0.nightly-2023-07-05-071214" \ --job-name periodic-ci-openshift-release-master-ci-4.15-upgrade-from-stable-4.14-e2e-aws-ovn-upgrade \ - --n 10 \ + --num-jobs 10 \ --jobs-file-path="/path/to/results/for/gangway/" ``` diff --git a/cmd/main.go b/cmd/main.go index 61df99f..63206a6 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "k8s.io/apimachinery/pkg/util/wait" + "github.com/spf13/cobra" "gopkg.in/yaml.v2" @@ -27,6 +29,12 @@ type JobRunIdentifier struct { JobRunID string } +type JobStatus struct { + BuildID string + JobURL string + JobID string +} + var opts struct { initial string latest string @@ -37,6 +45,13 @@ var opts struct { jobsFilePath string } +var backoff = wait.Backoff{ + Duration: 10 * time.Second, + Jitter: 0, + Factor: 2, + Steps: 100, +} + var cmd = &cobra.Command{ Use: "mycli", Short: "CLI tool to call an API", @@ -104,19 +119,31 @@ var cmd = &cobra.Command{ resp.Body.Close() // nolint // Get the job URL from prow easy access - jobURL, buildID, err := getJobURL(jobInfo.ID) + jobStatus := JobStatus{JobID: jobInfo.ID} + err = wait.ExponentialBackoff(backoff, func() (done bool, err error) { + err2 := jobStatus.getJobURL() + if err2 != nil { + return false, err2 + } + if len(jobStatus.JobURL) == 0 { + fmt.Println("Empty job URL will attempt retry") + return false, nil + } + return true, nil + }) + if err != nil { return err } jobRunIdentifier := JobRunIdentifier{ JobName: opts.jobName, - JobRunID: buildID, + JobRunID: jobStatus.BuildID, } jobRunIdentifiers = append(jobRunIdentifiers, jobRunIdentifier) // Print job info in tabular format - fmt.Printf(strFmt, strconv.Itoa(i+1), jobInfo.ID, jobURL) + fmt.Printf(strFmt, strconv.Itoa(i+1), jobInfo.ID, jobStatus.JobURL) // Sleep to avoid hitting the api too hard time.Sleep(time.Second) @@ -134,7 +161,7 @@ func NewCommand() *cobra.Command { cmd.Flags().StringArrayVarP(&opts.envVariables, "env", "e", nil, "Environment variables, VAR=VALUE") cmd.Flags().StringVarP(&opts.jobName, "job-name", "j", "", "Job name") cmd.Flags().StringVarP(&opts.apiURL, "api-url", "u", "", "API URL") - cmd.Flags().IntVarP(&opts.num, "n", "n", 1, fmt.Sprintf("Number of times to launch the job (max is %d)", maxJobs)) + cmd.Flags().IntVarP(&opts.num, "num-jobs", "n", 1, fmt.Sprintf("Number of times to launch the job (max is %d)", maxJobs)) cmd.Flags().StringVarP(&opts.jobsFilePath, "jobs-file-path", "p", "", "Save JobRunIdentifier JSON in the specified file path") return cmd @@ -162,55 +189,47 @@ func launchJob(appCIToken, apiURL string, data []byte) (*http.Response, error) { // getJobURL gets the url from prow so the user has a place to browse to see the // status of the prow job. Prow does not immediately have the prow job so we wait. -func getJobURL(jobID string) (string, string, error) { - const maxAttempts = 5 - const retryDelay = time.Second - url := "https://prow.ci.openshift.org/prowjob?prowjob=" + jobID +func (j *JobStatus) getJobURL() error { + url := "https://prow.ci.openshift.org/prowjob?prowjob=" + j.JobID var resp *http.Response var err error - for attempts := 0; attempts < maxAttempts; attempts++ { - resp, err = http.Get(url) - if err != nil { - fmt.Printf("Attempt %d: Error getting job URL: %v\n", attempts+1, err) - time.Sleep(retryDelay) - continue - } - body, err := io.ReadAll(resp.Body) - if err != nil { - resp.Body.Close() - return "", "", fmt.Errorf("error reading response body: %v", err) - } - resp.Body.Close() + resp, err = http.Get(url) + if err != nil { + fmt.Printf("Error getting job URL: %v\n", err) + return nil + } - // Search YAML document parts to find the section with status.url - documents := strings.Split(string(body), "---") - var statusURL, buildID string - for _, doc := range documents { - var jobInfo map[string]interface{} - if err := yaml.Unmarshal([]byte(doc), &jobInfo); err != nil { - continue - } - if status, ok := jobInfo["status"].(map[interface{}]interface{}); ok { + body, err := io.ReadAll(resp.Body) + if err != nil { + resp.Body.Close() + return fmt.Errorf("error reading response body: %v", err) + } + resp.Body.Close() - // try to get the build_id as well - // but only return when we have a valid URL - buildID = status["build_id"].(string) + // Search YAML document parts to find the section with status.url + documents := strings.Split(string(body), "---") + for _, doc := range documents { + var jobInfo map[string]interface{} + if err := yaml.Unmarshal([]byte(doc), &jobInfo); err != nil { + continue + } + if status, ok := jobInfo["status"].(map[interface{}]interface{}); ok { - if url, ok := status["url"].(string); ok { - statusURL = url - return statusURL, buildID, nil - } + // try to get the build_id as well + // but only return when we have a valid URL + if bid, ok := status["build_id"]; ok { + j.BuildID = bid.(string) } - } - if statusURL == "" { - time.Sleep(retryDelay) - continue + if url, ok := status["url"]; ok { + j.JobURL = url.(string) + return nil + } } - return statusURL, buildID, nil } - return "", "", fmt.Errorf("status.url not found in response after %d retries", maxAttempts) + + return nil } func outputJobRunIdentifiers(jobsFilePath string, jobRunIdentifiers []JobRunIdentifier) { @@ -224,7 +243,7 @@ func outputJobRunIdentifiers(jobsFilePath string, jobRunIdentifiers []JobRunIden // check to see if we end with path separator // if so set it to empty string delimiter := string(os.PathSeparator) - if len(opts.jobsFilePath) > 0 { + if len(jobsFilePath) > 0 { if strings.HasSuffix(jobsFilePath, delimiter) { delimiter = "" } diff --git a/go.mod b/go.mod index d2dee62..cc5ad84 100644 --- a/go.mod +++ b/go.mod @@ -2,10 +2,16 @@ module github.com/openshift-eng/gangway-cli go 1.20 -require github.com/spf13/cobra v1.7.0 +require ( + github.com/spf13/cobra v1.7.0 + gopkg.in/yaml.v2 v2.4.0 + k8s.io/apimachinery v0.27.2 +) require ( + github.com/go-logr/logr v1.2.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect ) diff --git a/go.sum b/go.sum index c050011..c40ba46 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -10,3 +13,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg= +k8s.io/apimachinery v0.27.2/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= +k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= From a825c714815c3659b7d41cd206918009844a05c4 Mon Sep 17 00:00:00 2001 From: Forrest Babcock Date: Tue, 24 Oct 2023 08:56:52 -0400 Subject: [PATCH 2/2] trt-1322: check if we have more runs before sleeping --- cmd/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 63206a6..4b004e7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -145,8 +145,11 @@ var cmd = &cobra.Command{ // Print job info in tabular format fmt.Printf(strFmt, strconv.Itoa(i+1), jobInfo.ID, jobStatus.JobURL) - // Sleep to avoid hitting the api too hard - time.Sleep(time.Second) + // if we are going to kick off another run + // sleep to avoid hitting the api too hard + if i < opts.num-1 { + time.Sleep(time.Second) + } } outputJobRunIdentifiers(opts.jobsFilePath, jobRunIdentifiers)