roachprod: secure clusters in dynamic admin url port

A recent change to support dynamic cluster port was introduced.
But, that was not considering secured clusters
This change supports secure cluster

Fixes: cockroachdb#117125
Epic: none

pkg/cmd/roachprod/flags.go

@@ -415,7 +415,7 @@ func initFlags() {
 		cmd.Flags().StringVarP(&config.Binary,
 			"binary", "b", config.Binary, "the remote cockroach binary to use")
 	}
-	for _, cmd := range []*cobra.Command{startCmd, startInstanceCmd, stopInstanceCmd, loadBalanceCmd, sqlCmd, pgurlCmd, adminurlCmd, runCmd, jaegerStartCmd, grafanaAnnotationCmd} {
+	for _, cmd := range []*cobra.Command{startCmd, startInstanceCmd, stopInstanceCmd, loadBalanceCmd, sqlCmd, pgurlCmd, adminurlCmd, runCmd, jaegerStartCmd, grafanaAnnotationCmd, updateTargetsCmd} {
 		// TODO(renato): remove --secure once the default of secure
 		// clusters has existed in roachprod long enough.
 		cmd.Flags().BoolVar(&secure,

pkg/cmd/roachprod/main.go

@@ -564,11 +564,14 @@ environment variables to the cockroach process.
 ` + tagHelp + `
 The default prometheus url is https://grafana.testeng.crdb.io/. This can be overwritten by using the
 environment variable COCKROACH_PROM_HOST_URL
+
+Note that if the cluster is started in insecure mode, set the insecure mode here as well by using the --insecure flag.
 `,
 	Args: cobra.ExactArgs(1),
 	Run: wrap(func(cmd *cobra.Command, args []string) error {
 		clusterSettingsOpts := []install.ClusterSettingOption{
 			install.TagOption(tag),
+			install.SecureOption(isSecure),
 			install.EnvOption(nodeEnv),
 		}
 		return roachprod.UpdateTargets(context.Background(), config.Logger, args[0], clusterSettingsOpts...)

pkg/roachprod/promhelperclient/client.go

@@ -67,7 +67,8 @@ func NewPromClient() *PromClient {
 // instanceConfigRequest is the HTTP request received for generating instance config
 type instanceConfigRequest struct {
 	//Config is the content of the yaml file
-	Config string `json:"config"`
+	Config   string `json:"config"`
+	Insecure bool   `json:"insecure"`
 }
 
 // UpdatePrometheusTargets updates the cluster config in the promUrl
@@ -76,9 +77,10 @@ func (c *PromClient) UpdatePrometheusTargets(
 	promUrl, clusterName string,
 	forceFetchCreds bool,
 	nodes []string,
+	insecure bool,
 	l *logger.Logger,
 ) error {
-	req, err := buildCreateRequest(nodes)
+	req, err := buildCreateRequest(nodes, insecure)
 	if err != nil {
 		return err
 	}
@@ -99,7 +101,7 @@ func (c *PromClient) UpdatePrometheusTargets(
 		defer func() { _ = response.Body.Close() }()
 		if response.StatusCode == http.StatusUnauthorized && !forceFetchCreds {
 			l.Printf("request failed - this may be due to a stale token. retrying with forceFetchCreds true ...")
-			return c.UpdatePrometheusTargets(ctx, promUrl, clusterName, true, nodes, l)
+			return c.UpdatePrometheusTargets(ctx, promUrl, clusterName, true, nodes, insecure, l)
 		}
 		body, err := io.ReadAll(response.Body)
 		if err != nil {
@@ -160,7 +162,7 @@ const clusterConfFileTemplate = `- targets:
 `
 
 // createClusterConfigFile creates the cluster config file per node
-func buildCreateRequest(nodes []string) (io.Reader, error) {
+func buildCreateRequest(nodes []string, insecure bool) (io.Reader, error) {
 	buffer := bytes.NewBufferString("---\n")
 	for i, n := range nodes {
 		if n == "" {
@@ -181,7 +183,7 @@ func buildCreateRequest(nodes []string) (io.Reader, error) {
 		}
 	}
 
-	b, err := json.Marshal(&instanceConfigRequest{Config: buffer.String()})
+	b, err := json.Marshal(&instanceConfigRequest{Config: buffer.String(), Insecure: insecure})
 	if err != nil {
 		return nil, err
 	}

pkg/roachprod/promhelperclient/client_test.go

@@ -48,7 +48,7 @@ func TestUpdatePrometheusTargets(t *testing.T) {
 				Body:       io.NopCloser(strings.NewReader("failed")),
 			}, nil
 		}
-		err := c.UpdatePrometheusTargets(ctx, promUrl, "c1", false, []string{"n1"}, l)
+		err := c.UpdatePrometheusTargets(ctx, promUrl, "c1", false, []string{"n1"}, true, l)
 		require.NotNil(t, err)
 		require.Equal(t, "request failed with status 400 and error failed", err.Error())
 	})
@@ -76,7 +76,7 @@ func TestUpdatePrometheusTargets(t *testing.T) {
 				StatusCode: 200,
 			}, nil
 		}
-		err := c.UpdatePrometheusTargets(ctx, promUrl, "c1", false, []string{"n1", "", "n3"}, l)
+		err := c.UpdatePrometheusTargets(ctx, promUrl, "c1", false, []string{"n1", "", "n3"}, true, l)
 		require.Nil(t, err)
 	})
 }

pkg/roachprod/promhelperclient/promhelper_utils.go

@@ -14,6 +14,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	secretmanager "cloud.google.com/go/secretmanager/apiv1"
@@ -22,8 +23,9 @@ import (
 )
 
 var (
+	userHome, _ = os.UserHomeDir()
 	// promCredFile is where the prom helper credentials are stored
-	promCredFile = os.TempDir() + "promhelpers-secrets"
+	promCredFile = filepath.Join(userHome, ".roachprod", "promhelper-secrets")
 )
 
 // FetchedFrom indicates where the credentials have been fetched from.

pkg/roachprod/roachprod.go

@@ -798,7 +798,7 @@ func updatePrometheusTargets(ctx context.Context, l *logger.Logger, c *install.S
 	if len(nodeIPPorts) > 0 {
 		if err := promhelperclient.NewPromClient().UpdatePrometheusTargets(ctx,
 			envutil.EnvOrDefaultString(prometheusHostUrlEnv, defaultPrometheusHostUrl),
-			c.Name, false, nodeIPPorts, l); err != nil {
+			c.Name, false, nodeIPPorts, !c.Secure, l); err != nil {
 			l.Errorf("creating cluster config failed for the ip:ports %v: %v", nodeIPPorts, err)
 		}
 	}