diff --git a/ssh/agent.go b/ssh/agent.go index 8d6a365d..82dc194a 100644 --- a/ssh/agent.go +++ b/ssh/agent.go @@ -89,6 +89,7 @@ func (agent *agent) Stop() error { logrus.Debugf("stopping the ssh-agent with Pid: %s", agent.Pid) p := echo.New().Env(agent.GetEnvVariables()).RunProc("ssh-agent -k") + logrus.Debugf("ssh-agent stopped: %s", p.Result()) return p.Err() } @@ -106,6 +107,7 @@ func StartAgent() (Agent, error) { return nil, fmt.Errorf("ssh-agent not found") } + logrus.Debugf("starting %s", sshAgentCmd) p := e.RunProc(fmt.Sprintf("%s -s", sshAgentCmd)) if p.Err() != nil { return nil, errors.Wrap(p.Err(), "failed to start ssh agent") @@ -119,6 +121,7 @@ func StartAgent() (Agent, error) { return nil, err } + logrus.Debugf("ssh-agent started %v", agentInfo) return agentFromInfo(agentInfo), nil } diff --git a/ssh/scp.go b/ssh/scp.go index d0c0e250..a83cd9fe 100644 --- a/ssh/scp.go +++ b/ssh/scp.go @@ -36,16 +36,16 @@ func CopyFrom(args SSHArgs, agent Agent, rootDir string, sourcePath string) erro return err } - sshCmd, err := makeSCPCmdStr(prog, args, sourcePath) + sshCmd, err := makeSCPCmdStr(prog, args) if err != nil { - return fmt.Errorf("scp: failed to build command string: %s", err) + return fmt.Errorf("scp: copyFrom: failed to build command string: %s", err) } - effectiveCmd := fmt.Sprintf(`%s %s`, sshCmd, targetPath) - logrus.Debug("scp: ", effectiveCmd) + effectiveCmd := fmt.Sprintf(`%s %s`, sshCmd, getCopyFromSourceTarget(args, sourcePath, targetPath)) + logrus.Debugf("scp: copFrom: cmd: [%s]", effectiveCmd) if agent != nil { - logrus.Debugf("Adding agent info: %s", agent.GetEnvVariables()) + logrus.Debugf("scp: copyFrom: adding agent info: %s", agent.GetEnvVariables()) e = e.Env(agent.GetEnvVariables()) } @@ -57,20 +57,69 @@ func CopyFrom(args SSHArgs, agent Agent, rootDir string, sourcePath string) erro if err := wait.ExponentialBackoff(retries, func() (bool, error) { p := e.RunProc(effectiveCmd) if p.Err() != nil { - logrus.Warn(fmt.Sprintf("scp: failed to connect to %s: error '%s %s': retrying connection", args.Host, p.Err(), p.Result())) + logrus.Warn(fmt.Sprintf("scp: copyFrom: failed to connect to %s: '%s %s': retrying connection", args.Host, p.Err(), p.Result())) return false, nil } return true, nil // worked }); err != nil { - logrus.Debugf("scp failed after %d tries", maxRetries) - return fmt.Errorf("scp: failed after %d attempt(s): %s", maxRetries, err) + return fmt.Errorf("scp: copyFrom: failed after %d attempt(s): %s", maxRetries, err) } - logrus.Debugf("scp: copied %s", sourcePath) + logrus.Debugf("scp: copyFrom: copied %s", sourcePath) return nil } -func makeSCPCmdStr(progName string, args SSHArgs, sourcePath string) (string, error) { +// CopyTo copies one or more files using SCP from local machine to +// remote host. +func CopyTo(args SSHArgs, agent Agent, sourcePath, targetPath string) error { + e := echo.New() + prog := e.Prog.Avail("scp") + if len(prog) == 0 { + return fmt.Errorf("scp program not found") + } + + if len(sourcePath) == 0 { + return fmt.Errorf("scp: copyTo: missing source path") + } + + if len(targetPath) == 0 { + return fmt.Errorf("scp: copyTo: missing target path") + } + + sshCmd, err := makeSCPCmdStr(prog, args) + if err != nil { + return fmt.Errorf("scp: copyTo: failed to build command string: %s", err) + } + + effectiveCmd := fmt.Sprintf(`%s %s`, sshCmd, getCopyToSourceTarget(args, sourcePath, targetPath)) + logrus.Debugf("scp: copyTo: cmd: [%s]", effectiveCmd) + + if agent != nil { + logrus.Debugf("scp: adding agent info: %s", agent.GetEnvVariables()) + e = e.Env(agent.GetEnvVariables()) + } + + maxRetries := args.MaxRetries + if maxRetries == 0 { + maxRetries = 10 + } + retries := wait.Backoff{Steps: maxRetries, Duration: time.Millisecond * 80, Jitter: 0.1} + if err := wait.ExponentialBackoff(retries, func() (bool, error) { + p := e.RunProc(effectiveCmd) + if p.Err() != nil { + logrus.Warn(fmt.Sprintf("scp: failed to connect to %s: '%s %s': retrying connection", args.Host, p.Err(), p.Result())) + return false, nil + } + return true, nil // worked + }); err != nil { + return fmt.Errorf("scp: copyTo: failed after %d attempt(s): %s", maxRetries, err) + } + + logrus.Debugf("scp: copyTo: copied %s -> %s", sourcePath, targetPath) + return nil +} + +func makeSCPCmdStr(progName string, args SSHArgs) (string, error) { if args.User == "" { return "", fmt.Errorf("scp: user is required") } @@ -111,8 +160,16 @@ func makeSCPCmdStr(progName string, args SSHArgs, sourcePath string) (string, er // build command as // scp -i -P -J user@host:path cmd := fmt.Sprintf( - `%s %s %s %s %s@%s:%s`, - scpCmdPrefix(), pkPath(), port(), proxyJump(), args.User, args.Host, sourcePath, + `%s %s %s %s`, + scpCmdPrefix(), pkPath(), port(), proxyJump(), ) return cmd, nil } + +func getCopyFromSourceTarget(args SSHArgs, sourcePath, targetPath string) string { + return fmt.Sprintf("%s@%s:%s %s", args.User, args.Host, sourcePath, targetPath) +} + +func getCopyToSourceTarget(args SSHArgs, sourcePath, targetPath string) string { + return fmt.Sprintf("%s %s@%s:%s", sourcePath, args.User, args.Host, targetPath) +} diff --git a/ssh/scp_test.go b/ssh/scp_test.go index 1f009a6a..4f396037 100644 --- a/ssh/scp_test.go +++ b/ssh/scp_test.go @@ -7,10 +7,11 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "testing" ) -func TestCopy(t *testing.T) { +func TestCopyFrom(t *testing.T) { tests := []struct { name string sshArgs SSHArgs @@ -45,13 +46,13 @@ func TestCopy(t *testing.T) { t.Run(test.name, func(t *testing.T) { defer func() { for file := range test.remoteFiles { - RemoveTestSSHFile(t, test.sshArgs, file) + RemoveRemoteTestSSHFile(t, test.sshArgs, file) } }() - // setup fake remote files + // setup fake files for file, content := range test.remoteFiles { - MakeTestSSHFile(t, test.sshArgs, file, content) + MakeRemoteTestSSHFile(t, test.sshArgs, file, content) } if err := CopyFrom(test.sshArgs, nil, support.TmpDirRoot(), test.srcFile); err != nil { @@ -82,54 +83,114 @@ func TestCopy(t *testing.T) { } } -// -//func TestMakeSCPCmdStr(t *testing.T) { -// tests := []struct { -// name string -// args SSHArgs -// cmdStr string -// source string -// shouldFail bool -// }{ -// { -// name: "user and host", -// args: SSHArgs{User: "sshuser", Host: "local.host"}, -// source: "/tmp/any", -// cmdStr: "scp -rpq -o StrictHostKeyChecking=no -P 22 sshuser@local.host:/tmp/any", -// }, -// { -// name: "user host and pkpath", -// args: SSHArgs{User: "sshuser", Host: "local.host", PrivateKeyPath: "/pk/path"}, -// source: "/foo/bar", -// cmdStr: "scp -rpq -o StrictHostKeyChecking=no -i /pk/path -P 22 sshuser@local.host:/foo/bar", -// }, -// { -// name: "user host pkpath and proxy", -// args: SSHArgs{User: "sshuser", Host: "local.host", PrivateKeyPath: "/pk/path", ProxyJump: &ProxyJumpArgs{User: "juser", Host: "jhost"}}, -// source: "userFile", -// cmdStr: "scp -rpq -o StrictHostKeyChecking=no -i /pk/path -P 22 -J juser@jhost sshuser@local.host:userFile", -// }, -// { -// name: "missing host", -// args: SSHArgs{User: "sshuser"}, -// shouldFail: true, -// }, -// } -// -// for _, test := range tests { -// t.Run(test.name, func(t *testing.T) { -// result, err := makeSCPCmdStr("scp", test.args, test.source) -// if err != nil && !test.shouldFail { -// t.Fatal(err) -// } -// cmdFields := strings.Fields(test.cmdStr) -// resultFields := strings.Fields(result) -// -// for i := range cmdFields { -// if cmdFields[i] != resultFields[i] { -// t.Fatalf("unexpected command string element: %s vs. %s", cmdFields, resultFields) -// } -// } -// }) -// } -//} +func TestCopyTo(t *testing.T) { + tests := []struct { + name string + sshArgs SSHArgs + localFiles map[string]string + file string + fileContent string + }{ + { + name: "copy single file to remote", + sshArgs: testSSHArgs, + localFiles: map[string]string{"local-foo.txt": "FooBar"}, + file: "local-foo.txt", + fileContent: "FooBar", + }, + { + name: "copy single file in dir to remote", + sshArgs: testSSHArgs, + localFiles: map[string]string{"local-foo/local-bar.txt": "FooBar"}, + file: "local-foo/local-bar.txt", + fileContent: "FooBar", + }, + { + name: "copy dir entire dir to remote", + sshArgs: testSSHArgs, + localFiles: map[string]string{"local-bar/local-foo.csv": "FooBar", "local-bar/local-bar.txt": "BarBar"}, + file: "local-bar/", + fileContent: "FooBar", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + defer func() { + for file := range test.localFiles { + RemoveLocalTestFile(t, filepath.Join(support.TmpDirRoot(), file)) + RemoveRemoteTestSSHFile(t, test.sshArgs, file) + } + }() + + // setup fake local files + for file, content := range test.localFiles { + MakeLocalTestFile(t, filepath.Join(support.TmpDirRoot(), file), content) + } + + // create remote dir if needed + // setup remote dir if needed + MakeRemoteTestSSHDir(t, test.sshArgs, test.file) + + sourceFile := filepath.Join(support.TmpDirRoot(), test.file) + if err := CopyTo(test.sshArgs, nil, sourceFile, test.file); err != nil { + t.Fatal(err) + } + + // validate copied files/dir + AssertRemoteTestSSHFile(t, test.sshArgs, test.file) + + }) + } +} + +func TestMakeSCPCmdStr(t *testing.T) { + tests := []struct { + name string + args SSHArgs + cmdStr string + source string + shouldFail bool + }{ + { + name: "default", + args: SSHArgs{User: "sshuser", Host: "local.host"}, + source: "/tmp/any", + cmdStr: "scp -rpq -o StrictHostKeyChecking=no -P 22", + }, + { + name: "pkpath", + args: SSHArgs{User: "sshuser", Host: "local.host", PrivateKeyPath: "/pk/path"}, + source: "/foo/bar", + cmdStr: "scp -rpq -o StrictHostKeyChecking=no -i /pk/path -P 22", + }, + { + name: "pkpath and proxy", + args: SSHArgs{User: "sshuser", Host: "local.host", PrivateKeyPath: "/pk/path", ProxyJump: &ProxyJumpArgs{User: "juser", Host: "jhost"}}, + source: "userFile", + cmdStr: "scp -rpq -o StrictHostKeyChecking=no -i /pk/path -P 22 -J juser@jhost", + }, + { + name: "missing host", + args: SSHArgs{User: "sshuser"}, + shouldFail: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := makeSCPCmdStr("scp", test.args) + if err != nil && !test.shouldFail { + t.Fatal(err) + } + cmdFields := strings.Fields(test.cmdStr) + resultFields := strings.Fields(result) + + for i := range cmdFields { + if cmdFields[i] != resultFields[i] { + t.Fatalf("unexpected command string element: %s vs. %s", cmdFields, resultFields) + } + } + }) + } +} diff --git a/ssh/test_support.go b/ssh/test_support.go index 483ce2dd..396b2d88 100644 --- a/ssh/test_support.go +++ b/ssh/test_support.go @@ -12,63 +12,86 @@ import ( "testing" ) -//////func mountTestSSHFile(t *testing.T, mountDir, fileName, content string) { -////// srcDir := filepath.Dir(fileName) -////// if len(srcDir) > 0 && srcDir != "." { -////// mountTestSSHDir(t, mountDir, srcDir) -////// } -////// -////// filePath := filepath.Join(mountDir, fileName) -////// t.Logf("mounting test file in SSH: %s", filePath) -////// if err := ioutil.WriteFile(filePath, []byte(content), 0644); err != nil { -////// t.Fatal(err) -////// } -//////} -//// -////func mountTestSSHDir(t *testing.T, mountDir, dir string) { -//// t.Logf("mounting dir in SSH: %s", dir) -//// mountPath := filepath.Join(mountDir, dir) -//// if err := os.MkdirAll(mountPath, 0754); err != nil && !os.IsExist(err) { -//// t.Fatal(err) -//// } -////} -// -//func removeTestSSHFile(t *testing.T, mountDir, fileName string) { -// t.Logf("removing file mounted in SSH: %s", fileName) -// filePath := filepath.Join(mountDir, fileName) -// if err := os.RemoveAll(filePath); err != nil && !os.IsNotExist(err) { -// t.Fatal(err) -// } -//} +func makeLocalTestDir(t *testing.T, dir string) { + t.Logf("creating local test dir: %s", dir) + if err := os.MkdirAll(dir, 0744); err != nil && !os.IsExist(err) { + t.Fatalf("makeLocalTestDir: failed to create dir: %s", err) + } + t.Logf("dir created: %s", dir) +} + +func MakeLocalTestFile(t *testing.T, filePath, content string) { + srcDir := filepath.Dir(filePath) + if len(srcDir) > 0 && srcDir != "." { + makeLocalTestDir(t, srcDir) + } + + t.Logf("creating test file: %s", filePath) + file, err := os.Create(filePath) + if err != nil { + t.Fatalf("MakeLocalTestFile: failed to create file: %s", err) + } + defer file.Close() + buf := bytes.NewBufferString(content) + if _, err := buf.WriteTo(file); err != nil { + t.Fatal(err) + } +} -func makeTestSSHDir(t *testing.T, args SSHArgs, dir string) { +func RemoveLocalTestFile(t *testing.T, fileName string) { + t.Logf("removing local test path: %s", fileName) + if err := os.RemoveAll(fileName); err != nil && !os.IsNotExist(err) { + t.Fatalf("RemoveLocalTestFile: failed: %s", err) + } +} + +func makeRemoteTestSSHDir(t *testing.T, args SSHArgs, dir string) { t.Logf("creating test dir over SSH: %s", dir) + if result, err := Run(args, nil, fmt.Sprintf("stat %s", dir)); err == nil { + t.Logf("remote dir already exist: %s", result) + return // already there + } _, err := Run(args, nil, fmt.Sprintf(`mkdir -p %s`, dir)) if err != nil { - t.Fatal(err) + t.Fatalf("makeRemoteTestSSHDir: failed: %s", err) } // validate - result, _ := Run(args, nil, fmt.Sprintf(`ls %s`, dir)) + result, err := Run(args, nil, fmt.Sprintf(`ls %s`, dir)) + if err != nil { + t.Fatalf("makeRemoteTestSSHDir %s", err) + } t.Logf("dir created: %s", result) } -func MakeTestSSHFile(t *testing.T, args SSHArgs, filePath, content string) { - srcDir := filepath.Dir(filePath) - if len(srcDir) > 0 && srcDir != "." { - makeTestSSHDir(t, args, srcDir) - } +func MakeRemoteTestSSHFile(t *testing.T, args SSHArgs, filePath, content string) { + MakeRemoteTestSSHDir(t, args, filePath) t.Logf("creating test file over SSH: %s", filePath) _, err := Run(args, nil, fmt.Sprintf(`echo '%s' > %s`, content, filePath)) if err != nil { - t.Fatal(err) + t.Fatalf("MakeRemoteTestSSHFile: failed: %s", err) } result, _ := Run(args, nil, fmt.Sprintf(`ls %s`, filePath)) t.Logf("file created: %s", result) } -func RemoveTestSSHFile(t *testing.T, args SSHArgs, fileName string) { +func MakeRemoteTestSSHDir(t *testing.T, args SSHArgs, filePath string) { + dir := filepath.Dir(filePath) + if len(dir) > 0 && dir != "." { + makeRemoteTestSSHDir(t, args, dir) + } +} + +func AssertRemoteTestSSHFile(t *testing.T, args SSHArgs, filePath string) { + t.Logf("stat remote SSH test file: %s", filePath) + _, err := Run(args, nil, fmt.Sprintf(`stat %s`, filePath)) + if err != nil { + t.Fatal(err) + } +} + +func RemoveRemoteTestSSHFile(t *testing.T, args SSHArgs, fileName string) { t.Logf("removing test file over SSH: %s", fileName) _, err := Run(args, nil, fmt.Sprintf(`rm -rf %s`, fileName)) if err != nil { diff --git a/starlark/copy_from.go b/starlark/copy_from.go index 26b0dfef..cdbcf0fc 100644 --- a/starlark/copy_from.go +++ b/starlark/copy_from.go @@ -66,7 +66,7 @@ func copyFromFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tu } } - results, err := execCopy(workdir, sourcePath, agent, resources) + results, err := execCopyFrom(workdir, sourcePath, agent, resources) if err != nil { return starlark.None, fmt.Errorf("%s: %s", identifiers.copyFrom, err) } @@ -83,7 +83,7 @@ func copyFromFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tu return starlark.NewList(resultList), nil } -func execCopy(rootPath string, path string, agent ssh.Agent, resources *starlark.List) ([]commandResult, error) { +func execCopyFrom(rootPath string, path string, agent ssh.Agent, resources *starlark.List) ([]commandResult, error) { if resources == nil { return nil, fmt.Errorf("%s: missing resources", identifiers.copyFrom) } @@ -117,7 +117,7 @@ func execCopy(rootPath string, path string, agent ssh.Agent, resources *starlark switch { case string(kind) == identifiers.hostResource && string(transport) == "ssh": - result, err := execCopySCP(host, rootDir, path, agent, res) + result, err := execSCPCopyFrom(host, rootDir, path, agent, res) if err != nil { logrus.Errorf("%s: failed to copyFrom %s: %s", identifiers.copyFrom, path, err) } @@ -131,7 +131,7 @@ func execCopy(rootPath string, path string, agent ssh.Agent, resources *starlark return results, nil } -func execCopySCP(host, rootDir, path string, agent ssh.Agent, res *starlarkstruct.Struct) (commandResult, error) { +func execSCPCopyFrom(host, rootDir, path string, agent ssh.Agent, res *starlarkstruct.Struct) (commandResult, error) { sshCfg := starlarkstruct.FromKeywords(starlarkstruct.Default, makeDefaultSSHConfig()) if val, err := res.Attr(identifiers.sshCfg); err == nil { if cfg, ok := val.(*starlarkstruct.Struct); ok { diff --git a/starlark/copy_from_test.go b/starlark/copy_from_test.go index 41c704e0..c66645e3 100644 --- a/starlark/copy_from_test.go +++ b/starlark/copy_from_test.go @@ -17,7 +17,7 @@ import ( "github.com/vmware-tanzu/crash-diagnostics/ssh" ) -func testCopyFuncForHostResources(t *testing.T, port, privateKey, username string) { +func testCopyFromFuncForHostResources(t *testing.T, port, privateKey, username string) { tests := []struct { name string remoteFiles map[string]string @@ -207,11 +207,11 @@ func testCopyFuncForHostResources(t *testing.T, port, privateKey, username strin for _, test := range tests { t.Run(test.name, func(t *testing.T) { for file, content := range test.remoteFiles { - ssh.MakeTestSSHFile(t, sshArgs, file, content) + ssh.MakeRemoteTestSSHFile(t, sshArgs, file, content) } defer func() { for file := range test.remoteFiles { - ssh.RemoveTestSSHFile(t, sshArgs, file) + ssh.RemoveRemoteTestSSHFile(t, sshArgs, file) } }() @@ -220,7 +220,7 @@ func testCopyFuncForHostResources(t *testing.T, port, privateKey, username strin } } -func testCopyFuncScriptForHostResources(t *testing.T, port, privateKey, username string) { +func testCopyFromFuncScriptForHostResources(t *testing.T, port, privateKey, username string) { tests := []struct { name string remoteFiles map[string]string @@ -357,11 +357,11 @@ result = cp(hosts)`, username, port, privateKey), sshArgs := ssh.SSHArgs{User: username, Host: "127.0.0.1", Port: port, PrivateKeyPath: privateKey} for _, test := range tests { for file, content := range test.remoteFiles { - ssh.MakeTestSSHFile(t, sshArgs, file, content) + ssh.MakeRemoteTestSSHFile(t, sshArgs, file, content) } defer func() { for file := range test.remoteFiles { - ssh.RemoveTestSSHFile(t, sshArgs, file) + ssh.RemoveRemoteTestSSHFile(t, sshArgs, file) } }() @@ -371,7 +371,7 @@ result = cp(hosts)`, username, port, privateKey), } } -func TestCopyFuncSSHAll(t *testing.T) { +func TestCopyFromFuncSSHAll(t *testing.T) { port := testSupport.PortValue() username := testSupport.CurrentUsername() privateKey := testSupport.PrivateKeyPath() @@ -380,8 +380,8 @@ func TestCopyFuncSSHAll(t *testing.T) { name string test func(t *testing.T, port, privateKey, username string) }{ - {name: "copyFrom func for host resources", test: testCopyFuncForHostResources}, - {name: "copy_from script for host resources", test: testCopyFuncScriptForHostResources}, + {name: "copyFrom func for host resources", test: testCopyFromFuncForHostResources}, + {name: "copy_from script for host resources", test: testCopyFromFuncScriptForHostResources}, } for _, test := range tests { diff --git a/starlark/copy_to.go b/starlark/copy_to.go new file mode 100644 index 00000000..8d4ca550 --- /dev/null +++ b/starlark/copy_to.go @@ -0,0 +1,141 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package starlark + +import ( + "errors" + "fmt" + + "github.com/sirupsen/logrus" + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" + + "github.com/vmware-tanzu/crash-diagnostics/ssh" +) + +// copyToFunc is a built-in starlark function that copies file resources from +// the local machine to a specified location on remote compute resources. +// +// If only one argument is provided, it is assumed to be the of file to copy. +// If resources are not provded, copy_to will search the starlark context for one. +// +// Starlark format: copy_to([] [,source_path=, target_path=, resources=resources]) + +func copyToFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var sourcePath, targetPath string + var resources *starlark.List + + if err := starlark.UnpackArgs( + identifiers.copyTo, args, kwargs, + "source_path", &sourcePath, + "target_path?", &targetPath, + "resources?", &resources, + ); err != nil { + return starlark.None, fmt.Errorf("%s: %s", identifiers.copyTo, err) + } + + if len(sourcePath) == 0 { + return starlark.None, fmt.Errorf("%s: path arg not set", identifiers.copyTo) + } + if len(targetPath) == 0 { + targetPath = sourcePath + } + + if resources == nil { + res, err := getResourcesFromThread(thread) + if err != nil { + return starlark.None, fmt.Errorf("%s: %s", identifiers.copyTo, err) + } + resources = res + } + + var agent ssh.Agent + var ok bool + if agentVal := thread.Local(identifiers.sshAgent); agentVal != nil { + agent, ok = agentVal.(ssh.Agent) + if !ok { + return starlark.None, errors.New("unable to fetch ssh-agent") + } + } + + results, err := execCopyTo(sourcePath, targetPath, agent, resources) + if err != nil { + return starlark.None, fmt.Errorf("%s: %s", identifiers.copyTo, err) + } + + // build list of struct as result + var resultList []starlark.Value + for _, result := range results { + if len(results) == 1 { + return result.toStarlarkStruct(), nil + } + resultList = append(resultList, result.toStarlarkStruct()) + } + + return starlark.NewList(resultList), nil +} + +func execCopyTo(sourcePath, targetPath string, agent ssh.Agent, resources *starlark.List) ([]commandResult, error) { + if resources == nil { + return nil, fmt.Errorf("%s: missing resources", identifiers.copyFrom) + } + + var results []commandResult + for i := 0; i < resources.Len(); i++ { + val := resources.Index(i) + res, ok := val.(*starlarkstruct.Struct) + if !ok { + return nil, fmt.Errorf("%s: unexpected resource type", identifiers.copyFrom) + } + + val, err := res.Attr("kind") + if err != nil { + return nil, fmt.Errorf("%s: resource.kind: %s", identifiers.copyFrom, err) + } + kind := val.(starlark.String) + + val, err = res.Attr("transport") + if err != nil { + return nil, fmt.Errorf("%s: resource.transport: %s", identifiers.copyFrom, err) + } + transport := val.(starlark.String) + + val, err = res.Attr("host") + if err != nil { + return nil, fmt.Errorf("%s: resource.host: %s", identifiers.copyFrom, err) + } + host := string(val.(starlark.String)) + + switch { + case string(kind) == identifiers.hostResource && string(transport) == "ssh": + result, err := execSCPCopyTo(host, sourcePath, targetPath, agent, res) + if err != nil { + logrus.Errorf("%s: failed to copy to : %s: %s", identifiers.copyTo, sourcePath, err) + } + results = append(results, result) + default: + logrus.Errorf("%s: unsupported or invalid resource kind: %s", identifiers.copyFrom, kind) + continue + } + } + + return results, nil +} + +func execSCPCopyTo(host, sourcePath, targetPath string, agent ssh.Agent, res *starlarkstruct.Struct) (commandResult, error) { + sshCfg := starlarkstruct.FromKeywords(starlarkstruct.Default, makeDefaultSSHConfig()) + if val, err := res.Attr(identifiers.sshCfg); err == nil { + if cfg, ok := val.(*starlarkstruct.Struct); ok { + sshCfg = cfg + } + } + + args, err := getSSHArgsFromCfg(sshCfg) + if err != nil { + return commandResult{}, err + } + args.Host = host + err = ssh.CopyTo(args, agent, sourcePath, targetPath) + return commandResult{resource: args.Host, result: targetPath, err: err}, err +} diff --git a/starlark/copy_to_test.go b/starlark/copy_to_test.go new file mode 100644 index 00000000..a9c82cd3 --- /dev/null +++ b/starlark/copy_to_test.go @@ -0,0 +1,360 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package starlark + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" + + "github.com/vmware-tanzu/crash-diagnostics/ssh" +) + +func testCopyToFuncForHostResources(t *testing.T, port, privateKey, username string) { + tests := []struct { + name string + localFiles map[string]string + kwargs func(t *testing.T) []starlark.Tuple + eval func(t *testing.T, kwargs []starlark.Tuple, sshArgs ssh.SSHArgs) + }{ + { + name: "single machine single file", + localFiles: map[string]string{"foo.txt": "FooBar"}, + kwargs: func(t *testing.T) []starlark.Tuple { + sshCfg := makeTestSSHConfig(privateKey, port, username) + resources := starlark.NewList([]starlark.Value{ + makeTestSSHHostResource("127.0.0.1", sshCfg), + }) + localFile := starlark.String(filepath.Join(testSupport.TmpDirRoot(), "foo.txt")) + remoteFile := starlark.String("foo.txt") + return []starlark.Tuple{ + []starlark.Value{starlark.String("resources"), resources}, + []starlark.Value{starlark.String("source_path"), localFile}, + []starlark.Value{starlark.String("target_path"), remoteFile}, + } + }, + + eval: func(t *testing.T, kwargs []starlark.Tuple, sshArgs ssh.SSHArgs) { + val, err := copyToFunc(newTestThreadLocal(t), nil, nil, kwargs) + if err != nil { + t.Fatal(err) + } + + var cpErr string + var targetPath string + if strct, ok := val.(*starlarkstruct.Struct); ok { + t.Logf(" starlarkstruct [%#v]", strct) + + if val, err := strct.Attr("err"); err == nil { + if r, ok := val.(starlark.String); ok { + cpErr = string(r) + } + } + if val, err := strct.Attr("result"); err == nil { + if r, ok := val.(starlark.String); ok { + targetPath = string(r) + } + } + } + + ssh.AssertRemoteTestSSHFile(t, sshArgs, targetPath) + + if cpErr != "" { + t.Fatal(cpErr) + } + }, + }, + + { + name: "multiple machines single files", + localFiles: map[string]string{"bar/bar.txt": "BarBar", "bar/foo.txt": "FooBar", "baz.txt": "BazBuz"}, + kwargs: func(t *testing.T) []starlark.Tuple { + sshCfg := makeTestSSHConfig(privateKey, port, username) + resources := starlark.NewList([]starlark.Value{ + makeTestSSHHostResource("localhost", sshCfg), + makeTestSSHHostResource("127.0.0.1", sshCfg), + }) + + localFile := starlark.String(filepath.Join(testSupport.TmpDirRoot(), "bar/bar.txt")) + remoteFile := starlark.String("bar/bar.txt") + + return []starlark.Tuple{ + []starlark.Value{starlark.String("resources"), resources}, + []starlark.Value{starlark.String("source_path"), localFile}, + []starlark.Value{starlark.String("target_path"), remoteFile}, + } + }, + eval: func(t *testing.T, kwargs []starlark.Tuple, sshArgs ssh.SSHArgs) { + val, err := copyToFunc(newTestThreadLocal(t), nil, nil, kwargs) + if err != nil { + t.Fatal(err) + } + + resultList, ok := val.(*starlark.List) + if !ok { + t.Fatalf("expecting type *starlark.List, got %T", val) + } + + for i := 0; i < resultList.Len(); i++ { + + var cpErr string + var targetPath string + if strct, ok := resultList.Index(i).(*starlarkstruct.Struct); ok { + if val, err := strct.Attr("err"); err == nil { + if r, ok := val.(starlark.String); ok { + cpErr = string(r) + } + } + if val, err := strct.Attr("result"); err == nil { + if r, ok := val.(starlark.String); ok { + targetPath = string(r) + } + } + } + + ssh.AssertRemoteTestSSHFile(t, sshArgs, targetPath) + + if cpErr != "" { + t.Fatal(cpErr) + } + } + }, + }, + + { + name: "multiple machines glob path", + localFiles: map[string]string{"bar/bar.txt": "BarBar", "bar/foo.txt": "FooBar", "bar/baz.csv": "BizzBuzz"}, + kwargs: func(t *testing.T) []starlark.Tuple { + sshCfg := makeTestSSHConfig(privateKey, port, username) + resources := starlark.NewList([]starlark.Value{ + makeTestSSHHostResource("127.0.0.1", sshCfg), + makeTestSSHHostResource("localhost", sshCfg), + }) + localFile := starlark.String(filepath.Join(testSupport.TmpDirRoot(), "bar/baz.csv")) + remoteFile := starlark.String("bar/baz.cvs") + + return []starlark.Tuple{ + []starlark.Value{starlark.String("resources"), resources}, + []starlark.Value{starlark.String("source_path"), localFile}, + []starlark.Value{starlark.String("target_path"), remoteFile}, + } + }, + eval: func(t *testing.T, kwargs []starlark.Tuple, sshArgs ssh.SSHArgs) { + val, err := copyToFunc(newTestThreadLocal(t), nil, nil, kwargs) + if err != nil { + t.Fatal(err) + } + + resultList, ok := val.(*starlark.List) + if !ok { + t.Fatalf("expecting type *starlark.List, got %T", val) + } + + for i := 0; i < resultList.Len(); i++ { + + var cpErr string + var targetPath string + if strct, ok := resultList.Index(i).(*starlarkstruct.Struct); ok { + if val, err := strct.Attr("err"); err == nil { + if r, ok := val.(starlark.String); ok { + cpErr = string(r) + } + } + if val, err := strct.Attr("result"); err == nil { + if r, ok := val.(starlark.String); ok { + targetPath = string(r) + } + } + } + + ssh.AssertRemoteTestSSHFile(t, sshArgs, targetPath) + + if cpErr != "" { + t.Fatal(cpErr) + } + } + }, + }, + } + + sshArgs := ssh.SSHArgs{User: username, Host: "127.0.0.1", Port: port, PrivateKeyPath: privateKey} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for file, content := range test.localFiles { + localFile := filepath.Join(testSupport.TmpDirRoot(), file) + ssh.MakeRemoteTestSSHDir(t, sshArgs, file) // if needed. + ssh.MakeLocalTestFile(t, localFile, content) + } + defer func() { + for file := range test.localFiles { + localFile := filepath.Join(testSupport.TmpDirRoot(), file) + ssh.RemoveRemoteTestSSHFile(t, sshArgs, file) + ssh.RemoveLocalTestFile(t, localFile) + } + }() + + test.eval(t, test.kwargs(t), sshArgs) + }) + } +} + +func testCopyToFuncScriptForHostResources(t *testing.T, port, privateKey, username string) { + tests := []struct { + name string + localFiles map[string]string + script string + eval func(t *testing.T, sshArgs ssh.SSHArgs, script string) + }{ + { + name: "multiple machines single copyTo", + localFiles: map[string]string{"foobar.c": "footext", "bar/bar.txt": "BarBar", "bar/foo.txt": "FooBar", "bar/baz.csv": "BizzBuzz"}, + script: fmt.Sprintf(` +set_defaults(resources(provider = host_list_provider(hosts=["127.0.0.1","localhost"], ssh_config = ssh_config(username="%s", port="%s", private_key_path="%s")))) +result = copy_to(source_path="%s/bar/foo.txt", target_path="bar/foo.txt")`, + username, port, privateKey, testSupport.TmpDirRoot()), + eval: func(t *testing.T, sshArgs ssh.SSHArgs, script string) { + exe := New() + if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { + t.Fatal(err) + } + + resultVal := exe.result["result"] + if resultVal == nil { + t.Fatal("copy_to() should be assigned to a variable") + } + resultList, ok := resultVal.(*starlark.List) + if !ok { + t.Fatalf("expecting type *starlark.List, got %T", resultVal) + } + + for i := 0; i < resultList.Len(); i++ { + + var cpErr string + var targetPath string + if strct, ok := resultList.Index(i).(*starlarkstruct.Struct); ok { + if val, err := strct.Attr("err"); err == nil { + if r, ok := val.(starlark.String); ok { + cpErr = string(r) + } + } + if val, err := strct.Attr("result"); err == nil { + if r, ok := val.(starlark.String); ok { + targetPath = string(r) + } + } + } + + ssh.AssertRemoteTestSSHFile(t, sshArgs, targetPath) + + if cpErr != "" { + t.Fatal(cpErr) + } + } + }, + }, + + { + name: "resource loop", + localFiles: map[string]string{"bar/bar.txt": "BarBar", "bar/foo.txt": "FooBar", "bar/baz.csv": "BizzBuzz"}, + script: fmt.Sprintf(` +# execute cmd on each host +def cp(hosts): + result = [] + for host in hosts: + result.append(copy_to(source_path="%s/bar/foo.txt", target_path="bar", resources=[host])) + return result + +# configuration +set_defaults(ssh_config(username="%s", port="%s", private_key_path="%s")) +hosts = resources(provider=host_list_provider(hosts=["127.0.0.1","localhost"])) +result = cp(hosts)`, testSupport.TmpDirRoot(), username, port, privateKey), + eval: func(t *testing.T, sshArgs ssh.SSHArgs, script string) { + exe := New() + if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { + t.Fatal(err) + } + + resultVal := exe.result["result"] + if resultVal == nil { + t.Fatal("capture() should be assigned to a variable") + } + resultList, ok := resultVal.(*starlark.List) + if !ok { + t.Fatalf("expecting type *starlark.List, got %T", resultVal) + } + + for i := 0; i < resultList.Len(); i++ { + + var cpErr string + var targetPath string + if strct, ok := resultList.Index(i).(*starlarkstruct.Struct); ok { + if val, err := strct.Attr("err"); err == nil { + if r, ok := val.(starlark.String); ok { + cpErr = string(r) + } + } + if val, err := strct.Attr("result"); err == nil { + if r, ok := val.(starlark.String); ok { + targetPath = string(r) + } + } + } + + ssh.AssertRemoteTestSSHFile(t, sshArgs, targetPath) + + if cpErr != "" { + t.Fatal(cpErr) + } + } + }, + }, + } + + sshArgs := ssh.SSHArgs{User: username, Host: "127.0.0.1", Port: port, PrivateKeyPath: privateKey} + for _, test := range tests { + for file, content := range test.localFiles { + localFile := filepath.Join(testSupport.TmpDirRoot(), file) + ssh.MakeRemoteTestSSHDir(t, sshArgs, file) // if needed. + ssh.MakeLocalTestFile(t, localFile, content) + } + + defer func() { + for file := range test.localFiles { + localFile := filepath.Join(testSupport.TmpDirRoot(), file) + ssh.RemoveRemoteTestSSHFile(t, sshArgs, file) + ssh.RemoveLocalTestFile(t, localFile) + } + }() + + t.Run(test.name, func(t *testing.T) { + test.eval(t, sshArgs, test.script) + }) + } +} + +func TestCopyToFuncSSHAll(t *testing.T) { + port := testSupport.PortValue() + username := testSupport.CurrentUsername() + privateKey := testSupport.PrivateKeyPath() + + tests := []struct { + name string + test func(t *testing.T, port, privateKey, username string) + }{ + {name: "copyToFunc for host resources", test: testCopyToFuncForHostResources}, + {name: "copy_from script for host resources", test: testCopyToFuncScriptForHostResources}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.test(t, port, privateKey, username) + defer os.RemoveAll(defaults.workdir) + }) + } +} diff --git a/starlark/run_test.go b/starlark/run_test.go index 62a0fdd8..f8cd284b 100644 --- a/starlark/run_test.go +++ b/starlark/run_test.go @@ -42,7 +42,7 @@ func testRunFuncHostResources(t *testing.T, port, privateKey, username string) { } } if expected != result { - t.Fatalf("runFunc returned unexpected value: %s", string(val.(starlark.String))) + t.Fatalf("runFunc returned unexpected value: %s", result) } }, }, @@ -73,7 +73,7 @@ func testRunFuncHostResources(t *testing.T, port, privateKey, username string) { } } if expected != result { - t.Fatalf("runFunc returned unexpected value: %s", string(val.(starlark.String))) + t.Fatalf("runFunc returned unexpected value: %s", result) } }, }, @@ -114,7 +114,7 @@ func testRunFuncHostResources(t *testing.T, port, privateKey, username string) { } } if expected != result { - t.Fatalf("runFunc returned unexpected value: %s", string(val.(starlark.String))) + t.Fatalf("runFunc returned unexpected value: %s", result) } } }, diff --git a/starlark/starlark_exec.go b/starlark/starlark_exec.go index 8326a4eb..9007f00f 100644 --- a/starlark/starlark_exec.go +++ b/starlark/starlark_exec.go @@ -106,6 +106,7 @@ func newPredeclareds() starlark.StringDict { identifiers.capture: starlark.NewBuiltin(identifiers.capture, captureFunc), identifiers.captureLocal: starlark.NewBuiltin(identifiers.capture, captureLocalFunc), identifiers.copyFrom: starlark.NewBuiltin(identifiers.copyFrom, copyFromFunc), + identifiers.copyTo: starlark.NewBuiltin(identifiers.copyTo, copyToFunc), identifiers.kubeCfg: starlark.NewBuiltin(identifiers.kubeCfg, KubeConfigFn), identifiers.kubeCapture: starlark.NewBuiltin(identifiers.kubeGet, KubeCaptureFn), identifiers.kubeGet: starlark.NewBuiltin(identifiers.kubeGet, KubeGetFn), diff --git a/starlark/support.go b/starlark/support.go index f2f2cb91..09d452a8 100644 --- a/starlark/support.go +++ b/starlark/support.go @@ -41,6 +41,7 @@ var ( capture string captureLocal string copyFrom string + copyTo string archive string os string setDefaults string @@ -75,6 +76,7 @@ var ( capture: "capture", captureLocal: "capture_local", copyFrom: "copy_from", + copyTo: "copy_to", archive: "archive", os: "os", setDefaults: "set_defaults",