diff --git a/Makefile b/Makefile index 253f3dfa..3ecc6078 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ build: scion-bat \ scion-netcat \ scion-sensorfetcher scion-sensorserver \ scion-ssh scion-sshd \ + scion-ftp scion-ftpd \ example-helloworld \ example-shttp-client example-shttp-server example-shttp-fileserver example-shttp-proxy @@ -90,6 +91,14 @@ scion-sshd: scion-webapp: go build -tags=$(TAGS) -o $(BIN)/$@ ./webapp/ +.PHONY: scion-ftp +scion-ftp: + go build -tags=$(TAGS) -o $(BIN)/$@ ./ftp/ + +.PHONY: scion-ftpd +scion-ftpd: + go build -tags=$(TAGS) -o $(BIN)/$@ ./ftpd/ + .PHONY: example-helloworld example-helloworld: go build -tags=$(TAGS) -o $(BIN)/$@ ./_examples/helloworld/ diff --git a/ftp/README.md b/ftp/README.md new file mode 100644 index 00000000..51541d74 --- /dev/null +++ b/ftp/README.md @@ -0,0 +1,14 @@ +# scion-ftp + +This is an interactive FTP client for testing and demonstrating usage of FTP on the SCION network. Build this +application from [scion-apps](../../) using the command `make scion-ftp` + +``` +$ scion-ftp +Usage of scion-ftp: + -hercules string + Enable RETR_HERCULES using the Hercules binary specified + In Hercules mode, scionFTP checks the following directories for Hercules config files: ., /etc, /etc/scion-ftp +``` + +After starting the application run `help` to see all available commands. diff --git a/ftp/internal/ftp/LICENSE b/ftp/internal/ftp/LICENSE new file mode 100644 index 00000000..9ab085c5 --- /dev/null +++ b/ftp/internal/ftp/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2011-2013, Julien Laffaye + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/ftp/internal/ftp/README.md b/ftp/internal/ftp/README.md new file mode 100644 index 00000000..3162c65b --- /dev/null +++ b/ftp/internal/ftp/README.md @@ -0,0 +1,24 @@ +# scionftp client library + +Client package for FTP + GridFTP extension, adapted to the SCION network. +Forked from [jlaffaye/ftp](https://github.com/jlaffaye/ftp), with SCION support originally added in [elwin/scionFTP](https://github.com/elwin/scionFTP). + +## Example ## + +```go +c, err := ftp.Dial("1-ff00:0:110,[127.0.0.1]:4000, 1-ff00:0:110,[127.0.0.1]:2121", ftp.DialWithTimeout(5*time.Second)) +if err != nil { + log.Fatal(err) +} + +err = c.Login("admin", "123456") +if err != nil { + log.Fatal(err) +} + +// Do something with the FTP connection + +if err := c.Quit(); err != nil { + log.Fatal(err) +} +``` diff --git a/ftp/internal/ftp/client_test.go b/ftp/internal/ftp/client_test.go new file mode 100644 index 00000000..9040b72b --- /dev/null +++ b/ftp/internal/ftp/client_test.go @@ -0,0 +1,281 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +// +// Copyright 2020-2021 ETH Zurich modifications to add support for SCION + +package ftp + +import ( + "bytes" + "io/ioutil" + "net/textproto" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +const ( + testData = "Just some text" + testDir = "mydir" +) + +func TestConnPASV(t *testing.T) { + testConn(t, true) +} + +func TestConnEPSV(t *testing.T) { + testConn(t, false) +} + +func testConn(t *testing.T, disableEPSV bool) { + + mock, c := openConn(t, "127.0.0.1", DialWithTimeout(5*time.Second), DialWithDisabledEPSV(disableEPSV)) + + err := c.Login("anonymous", "anonymous") + if err != nil { + t.Fatal(err) + } + + err = c.NoOp() + if err != nil { + t.Error(err) + } + + err = c.ChangeDir("incoming") + if err != nil { + t.Error(err) + } + + dir, err := c.CurrentDir() + if err != nil { + t.Error(err) + } else { + if dir != "/incoming" { + t.Error("Wrong dir: " + dir) + } + } + + data := bytes.NewBufferString(testData) + err = c.Stor("test", data) + if err != nil { + t.Error(err) + } + + _, err = c.List(".") + if err != nil { + t.Error(err) + } + + err = c.Rename("test", "tset") + if err != nil { + t.Error(err) + } + + // Read without deadline + r, err := c.Retr("tset") + if err != nil { + t.Error(err) + } else { + buf, err := ioutil.ReadAll(r) + if err != nil { + t.Error(err) + } + if string(buf) != testData { + t.Errorf("'%s'", buf) + } + assert.NoError(t, r.Close()) + assert.NoError(t, r.Close()) // test we can close two times + } + + // Read with deadline + r, err = c.Retr("tset") + if err != nil { + t.Error(err) + } else { + assert.NoError(t, r.SetDeadline(time.Now())) + _, err := ioutil.ReadAll(r) + if err == nil { + t.Error("deadline should have caused error") + } else if !strings.HasSuffix(err.Error(), "i/o timeout") { + t.Error(err) + } + assert.NoError(t, r.Close()) + } + + // Read with offset + r, err = c.RetrFrom("tset", 5) + if err != nil { + t.Error(err) + } else { + buf, err := ioutil.ReadAll(r) + if err != nil { + t.Error(err) + } + expected := testData[5:] + if string(buf) != expected { + t.Errorf("read %q, expected %q", buf, expected) + } + assert.NoError(t, r.Close()) + } + + fileSize, err := c.FileSize("magic-file") + if err != nil { + t.Error(err) + } + if fileSize != 42 { + t.Errorf("file size %q, expected %q", fileSize, 42) + } + + _, err = c.FileSize("not-found") + if err == nil { + t.Fatal("expected error, got nil") + } + + err = c.Delete("tset") + if err != nil { + t.Error(err) + } + + err = c.MakeDir(testDir) + if err != nil { + t.Error(err) + } + + err = c.ChangeDir(testDir) + if err != nil { + t.Error(err) + } + + err = c.ChangeDirToParent() + if err != nil { + t.Error(err) + } + + entries, err := c.NameList("/") + if err != nil { + t.Error(err) + } + if len(entries) != 1 || entries[0] != "/incoming" { + t.Errorf("Unexpected entries: %v", entries) + } + + err = c.RemoveDir(testDir) + if err != nil { + t.Error(err) + } + + err = c.Logout() + if err != nil { + if protoErr := err.(*textproto.Error); protoErr != nil { + if protoErr.Code != StatusNotImplemented { + t.Error(err) + } + } else { + t.Error(err) + } + } + + if err := c.Quit(); err != nil { + t.Fatal(err) + } + + // Wait for the connection to close + mock.Wait() + + err = c.NoOp() + if err == nil { + t.Error("Expected error") + } +} + +func TestTimeout(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + c, err := DialTimeout("localhost:2121", 1*time.Second) + if err == nil { + t.Fatal("expected timeout, got nil error") + _ = c.Quit() + } +} + +func TestWrongLogin(t *testing.T) { + mock, err := newFtpMock(t, "127.0.0.1") + if err != nil { + t.Fatal(err) + } + defer mock.Close() + + c, err := DialTimeout(mock.Addr(), 5*time.Second) + if err != nil { + t.Fatal(err) + } + defer func() { assert.NoError(t, c.Quit()) }() + + err = c.Login("zoo2Shia", "fei5Yix9") + if err == nil { + t.Fatal("expected error, got nil") + } +} + +func TestDeleteDirRecur(t *testing.T) { + mock, c := openConn(t, "127.0.0.1") + + err := c.RemoveDirRecur("testDir") + if err != nil { + t.Error(err) + } + + if err := c.Quit(); err != nil { + t.Fatal(err) + } + + // Wait for the connection to close + mock.Wait() +} + +// func TestFileDeleteDirRecur(t *testing.T) { +// mock, c := openConn(t, "127.0.0.1") + +// err := c.RemoveDirRecur("testFile") +// if err == nil { +// t.Fatal("expected error got nil") +// } + +// if err := c.Quit(); err != nil { +// t.Fatal(err) +// } + +// // Wait for the connection to close +// mock.Wait() +// } + +func TestMissingFolderDeleteDirRecur(t *testing.T) { + mock, c := openConn(t, "127.0.0.1") + + err := c.RemoveDirRecur("missing-dir") + if err == nil { + t.Fatal("expected error got nil") + } + + if err := c.Quit(); err != nil { + t.Fatal(err) + } + + // Wait for the connection to close + mock.Wait() +} diff --git a/ftp/internal/ftp/cmd.go b/ftp/internal/ftp/cmd.go new file mode 100644 index 00000000..217db177 --- /dev/null +++ b/ftp/internal/ftp/cmd.go @@ -0,0 +1,789 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +// +// Copyright 2019-2021 ETH Zurich modifications to add support for SCION + +package ftp + +import ( + "bufio" + "errors" + "fmt" + "io" + "log" + "net" + "net/textproto" + "os" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/scionproto/scion/go/lib/snet" + + "github.com/netsec-ethz/scion-apps/internal/ftp/hercules" + libmode "github.com/netsec-ethz/scion-apps/internal/ftp/mode" + "github.com/netsec-ethz/scion-apps/internal/ftp/socket" + "github.com/netsec-ethz/scion-apps/internal/ftp/striping" +) + +// Login authenticates the scionftp with specified user and password. +// +// "anonymous"/"anonymous" is a common user/password scheme for FTP servers +// that allows anonymous read-only accounts. +func (c *ServerConn) Login(user, password string) error { + code, message, err := c.cmd(-1, "USER %s", user) + if err != nil { + return err + } + + switch code { + case StatusLoggedIn: + case StatusUserOK: + _, _, err = c.cmd(StatusLoggedIn, "PASS %s", password) + if err != nil { + return err + } + default: + return errors.New(message) + } + + // Switch to binary mode + if _, _, err = c.cmd(StatusCommandOK, "TYPE I"); err != nil { + return err + } + + // Switch to UTF-8 + err = c.setUTF8() + + return err +} + +// feat issues a FEAT FTP command to list the additional commands supported by +// the remote FTP server. +// FEAT is described in RFC 2389 +func (c *ServerConn) feat() error { + code, message, err := c.cmd(-1, "FEAT") + if err != nil { + return err + } + + if code != StatusSystem { + // The server does not support the FEAT command. This is not an + // error: we consider that there is no additional feature. + return nil + } + + lines := strings.Split(message, "\n") + for _, line := range lines { + if !strings.HasPrefix(line, " ") { + continue + } + + line = strings.TrimSpace(line) + featureElements := strings.SplitN(line, " ", 2) + + command := featureElements[0] + + var commandDesc string + if len(featureElements) == 2 { + commandDesc = featureElements[1] + } + + c.features[command] = commandDesc + } + + return nil +} + +// setUTF8 issues an "OPTS UTF8 ON" command. +func (c *ServerConn) setUTF8() error { + if _, ok := c.features["UTF8"]; !ok { + return nil + } + + code, message, err := c.cmd(-1, "OPTS UTF8 ON") + if err != nil { + return err + } + + // Workaround for FTP servers, that does not support this option. + if code == StatusBadArguments { + return nil + } + + // The ftpd "filezilla-server" has FEAT support for UTF8, but always returns + // "202 UTF8 mode is always enabled. No need to send this command." when + // trying to use it. That's OK + if code == StatusCommandNotImplemented { + return nil + } + + if code != StatusCommandOK { + return errors.New(message) + } + + return nil +} + +// epsv issues an "EPSV" command to get a port number for a data connection. +func (c *ServerConn) epsv() (port int, err error) { + _, line, err := c.cmd(StatusExtendedPassiveMode, "EPSV") + if err != nil { + return + } + + start := strings.Index(line, "|||") + end := strings.LastIndex(line, "|") + if start == -1 || end == -1 { + err = errors.New("invalid EPSV response format") + return + } + port, err = strconv.Atoi(line[start+3 : end]) + return +} + +// pasv issues a "PASV" command to get a port number for a data connection. +func (c *ServerConn) pasv() (port int, err error) { + return c.epsv() +} + +// getDataConnPort returns a host, port for a new data connection +// it uses the best available method to do so +func (c *ServerConn) getDataConnPort() (int, error) { + return c.pasv() +} + +// TODO: Close connections if there is an error with the others +// openDataConn creates a new FTP data connection. +func (c *ServerConn) openDataConn() (net.Conn, error) { + + if c.mode == libmode.ExtendedBlockMode { + addrs, err := c.spas() + if err != nil { + return nil, err + } + + wg := &sync.WaitGroup{} + + sockets := make([]net.Conn, len(addrs)) + wg.Add(len(sockets)) + for i := range sockets { + + go func(i int) { + defer wg.Done() + + conn, err := socket.DialAddr(addrs[i]) + if err != nil { + log.Fatalf("failed to connect: %s", err) + } + + sockets[i] = conn + }(i) + } + + wg.Wait() + + return striping.NewMultiSocket(sockets, c.blockSize), nil + + } else { + // For Stream and Hercules mode, data connections work the same, + // except that Hercules will "steal" the traffic from the Kernel. + // Note that this only happens for file transfers, the remaining + // data connections will work as normal in stream mode. + port, err := c.getDataConnPort() + if err != nil { + return nil, err + } + + remote := c.socket.RemoteAddr().(*snet.UDPAddr).Copy() + remote.Host.Port = port + + conn, err := socket.DialAddr(remote.String()) + if err != nil { + return nil, err + } + + return conn, nil + } +} + +// cmd is a helper function to execute a command and check for the expected FTP +// return code +func (c *ServerConn) cmd(expected int, format string, args ...interface{}) (int, string, error) { + _, err := c.conn.Cmd(format, args...) + if err != nil { + return 0, "", err + } + + return c.conn.ReadResponse(expected) +} + +// cmdDataConnFrom executes a command which require a FTP data connection. +// Issues a REST FTP command to specify the number of bytes to skip for the transfer. +func (c *ServerConn) cmdDataConnFrom(offset uint64, format string, args ...interface{}) (net.Conn, error) { + conn, err := c.openDataConn() + if err != nil { + return nil, err + } + + if offset != 0 { + _, _, err := c.cmd(StatusRequestFilePending, "REST %d", offset) + if err != nil { + _ = conn.Close() + return nil, err + } + } + + _, err = c.conn.Cmd(format, args...) + if err != nil { + _ = conn.Close() + return nil, err + } + + code, msg, err := c.conn.ReadResponse(-1) + if err != nil { + _ = conn.Close() + return nil, err + } + if code != StatusAlreadyOpen && code != StatusAboutToSend { + _ = conn.Close() + return nil, &textproto.Error{Code: code, Msg: msg} + } + + return conn, nil +} + +// NameList issues an NLST FTP command. +func (c *ServerConn) NameList(path string) (entries []string, err error) { + conn, err := c.cmdDataConnFrom(0, "NLST %s", path) + if err != nil { + return + } + + r := &Response{conn: conn, c: c} + defer func() { _ = r.Close() }() + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + entries = append(entries, scanner.Text()) + } + if err = scanner.Err(); err != nil { + return entries, err + } + return +} + +func (c *ServerConn) IsMlstSupported() bool { + _, mlstSupported := c.features["MLST"] + return mlstSupported +} + +// List issues a LIST FTP command. +func (c *ServerConn) List(path string) (entries []*Entry, err error) { + + var cmd string + var parser parseFunc + + if c.IsMlstSupported() { + cmd = "MLSD" + parser = parseRFC3659ListLine + } else { + cmd = "LIST" + parser = parseListLine + } + + conn, err := c.cmdDataConnFrom(0, "%s %s", cmd, path) + if err != nil { + return + } + + r := &Response{conn: conn, c: c} + defer func() { _ = r.Close() }() + + scanner := bufio.NewScanner(r) + now := time.Now() + for scanner.Scan() { + entry, err := parser(scanner.Text(), now, c.options.location) + if err == nil { + entries = append(entries, entry) + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + return +} + +// ChangeDir issues a CWD FTP command, which changes the current directory to +// the specified path. +func (c *ServerConn) ChangeDir(path string) error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "CWD %s", path) + return err +} + +// ChangeDirToParent issues a CDUP FTP command, which changes the current +// directory to the parent directory. This is similar to a call to ChangeDir +// with a path set to "..". +func (c *ServerConn) ChangeDirToParent() error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "CDUP") + return err +} + +// CurrentDir issues a PWD FTP command, which Returns the path of the current +// directory. +func (c *ServerConn) CurrentDir() (string, error) { + _, msg, err := c.cmd(StatusPathCreated, "PWD") + if err != nil { + return "", err + } + + start := strings.Index(msg, "\"") + end := strings.LastIndex(msg, "\"") + + if start == -1 || end == -1 { + return "", errors.New("unsuported PWD response format") + } + + return msg[start+1 : end], nil +} + +// FileSize issues a SIZE FTP command, which Returns the size of the file +func (c *ServerConn) FileSize(path string) (int64, error) { + _, msg, err := c.cmd(StatusFile, "SIZE %s", path) + if err != nil { + return 0, err + } + + return strconv.ParseInt(msg, 10, 64) +} + +// Retr issues a RETR FTP command to fetch the specified file from the remote +// FTP server. +// +// The returned ReadCloser must be closed to cleanup the FTP data connection. +func (c *ServerConn) Retr(path string) (*Response, error) { + return c.RetrFrom(path, 0) +} + +// RetrFrom issues a RETR FTP command to fetch the specified file from the remote +// FTP server, the server will not send the offset first bytes of the file. +// +// The returned ReadCloser must be closed to cleanup the FTP data connection. +func (c *ServerConn) RetrFrom(path string, offset uint64) (*Response, error) { + conn, err := c.cmdDataConnFrom(offset, "RETR %s", path) + if err != nil { + return nil, err + } + + return &Response{conn: conn, c: c}, nil +} + +func (c *ServerConn) RetrHercules(herculesBinary, remotePath, localPath string) error { + ftpCmd := fmt.Sprintf("RETR %s", remotePath) + return c.herculesDownload(herculesBinary, localPath, ftpCmd, -1) +} + +func (c *ServerConn) RetrHerculesFrom(herculesBinary, remotePath, localPath string, offset int64) error { + _, _, err := c.cmd(StatusRequestFilePending, "REST %d", offset) + if err != nil { + return err + } + + ftpCmd := fmt.Sprintf("RETR %s", remotePath) + return c.herculesDownload(herculesBinary, localPath, ftpCmd, offset) +} + +func (c *ServerConn) herculesDownload(herculesBinary, localPath, ftpCmd string, offset int64) error { + if herculesBinary == "" { + return fmt.Errorf("you need to specify -hercules to use this feature") + } + herculesConfig, err := hercules.ResolveConfig() + if err != nil { + return err + } + if herculesConfig == nil { + log.Printf("No Hercules configuration found, using defaults (queue 0, copy mode)") + } else { + log.Printf("Using Hercules configuration at %s", *herculesConfig) + } + + // check file access as unprivileged user + fileCreated, err := hercules.AssertFileWriteable(localPath) + if err != nil { + return err + } + defer func() { + if err != nil && fileCreated { + err2 := syscall.Unlink(localPath) + if err2 != nil { + log.Printf("could not delete file: %s", err2) + } + } + }() + + sock, err := c.openDataConn() + if err != nil { + return err + } + defer func() { _ = sock.Close() }() + + cmd, err := hercules.PrepareHerculesRecvCommand(herculesBinary, herculesConfig, sock.LocalAddr().(*net.UDPAddr), localPath, offset) + if err != nil { + return err + } + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + log.Printf("run Hercules: %s", cmd) + err = cmd.Start() + if err != nil { + return fmt.Errorf("could not start Hercules: %s", err) + } + + code, _, err := c.cmd(StatusAboutToSend, ftpCmd) + if code != StatusAboutToSend { + err2 := cmd.Process.Kill() + if err2 != nil { + return fmt.Errorf("transfer failed: %s\ncould not stop Hercules: %s", err, err2) + } else { + return fmt.Errorf("transfer failed: %s", err) + } + } else { + _, msg, err := c.conn.ReadResponse(StatusClosingDataConnection) + log.Printf("%s", msg) + if err != nil { + err2 := cmd.Process.Kill() + if err2 != nil { + return fmt.Errorf("transfer failed: %s\ncould not stop Hercules: %s", err, err2) + } else { + return fmt.Errorf("transfer failed: %s", err) + } + } + err = cmd.Wait() + if err != nil { + return fmt.Errorf("error during transfer: %s", err) + } else { + return hercules.OwnFile(localPath) + } + } +} + +// Stor issues a STOR FTP command to store a file to the remote FTP server. +// Stor creates the specified file with the content of the io.Reader. +// +// Hint: io.Pipe() can be used if an io.Writer is required. +func (c *ServerConn) Stor(path string, r io.Reader) error { + return c.StorFrom(path, r, 0) +} + +// StorFrom issues a STOR FTP command to store a file to the remote FTP server. +// Stor creates the specified file with the content of the io.Reader, writing +// on the server will start at the given file offset. +// +// Hint: io.Pipe() can be used if an io.Writer is required. +func (c *ServerConn) StorFrom(path string, r io.Reader, offset uint64) error { + + conn, err := c.cmdDataConnFrom(offset, "STOR %s", path) + if err != nil { + return err + } + + n, err := io.Copy(conn, r) + + if err != nil { + return err + } else { + fmt.Printf("Wrote %d bytes\n", n) + } + + _ = conn.Close() // Needs to be before the statement below, otherwise deadlocks + _, _, err = c.conn.ReadResponse(StatusClosingDataConnection) + + return err +} + +func (c *ServerConn) StorHercules(herculesBinary, localPath, remotePath string) error { + ftpCmd := fmt.Sprintf("STOR %s", remotePath) + return c.uploadHercules(herculesBinary, localPath, ftpCmd, -1) +} + +func (c *ServerConn) uploadHercules(herculesBinary, localPath, ftpCmd string, offset int64) error { + if herculesBinary == "" { + return fmt.Errorf("you need to specify -hercules to use this feature") + } + herculesConfig, err := hercules.ResolveConfig() + if err != nil { + return err + } + if herculesConfig == nil { + log.Printf("No Hercules configuration found, using defaults (queue 0, copy mode)") + } else { + log.Printf("Using Hercules configuration at %s", *herculesConfig) + } + + f, err := os.Open(localPath) + if err != nil { + return err + } + _ = f.Close() + + sock, err := c.openDataConn() + if err != nil { + return err + } + defer func() { _ = sock.Close() }() + + code, _, err := c.cmd(StatusAlreadyOpen, ftpCmd) + if code != StatusAlreadyOpen { + return fmt.Errorf("transfer failed: %s", err) + } + + cmd, err := hercules.PrepareHerculesSendCommand(herculesBinary, herculesConfig, sock.LocalAddr().(*net.UDPAddr), sock.RemoteAddr().(*snet.UDPAddr), localPath, offset) + if err != nil { + return err + } + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + log.Printf("run Hercules: %s", cmd) + err = cmd.Start() + if err != nil { + return fmt.Errorf("could not start Hercules: %s", err) + } + + _, msg, err := c.conn.ReadResponse(StatusClosingDataConnection) + log.Printf("%s", msg) + if err != nil { + err2 := cmd.Process.Kill() + if err2 != nil { + return fmt.Errorf("transfer failed: %s\ncould not stop Hercules: %s", err, err2) + } else { + return fmt.Errorf("transfer failed: %s", err) + } + } + err = cmd.Wait() + if err != nil { + return fmt.Errorf("error during transfer: %s", err) + } else { + return hercules.OwnFile(localPath) + } +} + +// Rename renames a file on the remote FTP server. +func (c *ServerConn) Rename(from, to string) error { + _, _, err := c.cmd(StatusRequestFilePending, "RNFR %s", from) + if err != nil { + return err + } + + _, _, err = c.cmd(StatusRequestedFileActionOK, "RNTO %s", to) + return err +} + +// Delete issues a DELE FTP command to delete the specified file from the +// remote FTP server. +func (c *ServerConn) Delete(path string) error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "DELE %s", path) + return err +} + +// RemoveDirRecur deletes a non-empty folder recursively using +// RemoveDir and Delete +func (c *ServerConn) RemoveDirRecur(path string) error { + err := c.ChangeDir(path) + if err != nil { + return err + } + currentDir, err := c.CurrentDir() + if err != nil { + return err + } + + entries, err := c.List(currentDir) + if err != nil { + return err + } + + for _, entry := range entries { + if entry.Name != ".." && entry.Name != "." { + if entry.Type == EntryTypeFolder { + err = c.RemoveDirRecur(currentDir + "/" + entry.Name) + if err != nil { + return err + } + } else { + err = c.Delete(entry.Name) + if err != nil { + return err + } + } + } + } + err = c.ChangeDirToParent() + if err != nil { + return err + } + err = c.RemoveDir(currentDir) + return err +} + +// MakeDir issues a MKD FTP command to create the specified directory on the +// remote FTP server. +func (c *ServerConn) MakeDir(path string) error { + _, _, err := c.cmd(StatusPathCreated, "MKD %s", path) + return err +} + +// RemoveDir issues a RMD FTP command to remove the specified directory from +// the remote FTP server. +func (c *ServerConn) RemoveDir(path string) error { + _, _, err := c.cmd(StatusRequestedFileActionOK, "RMD %s", path) + return err +} + +// NoOp issues a NOOP FTP command. +// NOOP has no effects and is usually used to prevent the remote FTP server to +// close the otherwise idle connection. +func (c *ServerConn) NoOp() error { + _, err := c.conn.Cmd("NOOP") + if err != nil { + return err + } + + _, _, err = c.conn.ReadResponse(StatusCommandOK) + return err +} + +// Logout issues a REIN FTP command to logout the current user. +func (c *ServerConn) Logout() error { + _, _, err := c.cmd(StatusReady, "REIN") + return err +} + +// Quit issues a QUIT FTP command to properly close the connection from the +// remote FTP server. +func (c *ServerConn) Quit() error { + _, _ = c.conn.Cmd("QUIT") + return c.conn.Close() +} + +// GridFTP Extensions (https://www.ogf.org/documents/GFD.20.pdf) + +// Switch Mode +func (c *ServerConn) Mode(mode byte) error { + switch mode { // check if we support the requested mode + case libmode.Stream: + case libmode.ExtendedBlockMode: + case libmode.Hercules: + break + default: + return fmt.Errorf("unsupported mode: %v", mode) + } + + code, line, err := c.cmd(StatusCommandOK, "MODE %s", string(mode)) + if err != nil { + return fmt.Errorf("failed to set Mode %v: %d - %s", mode, code, line) + } + + c.mode = mode + return nil +} + +func (c *ServerConn) IsModeHercules() bool { + return c.mode == libmode.Hercules +} + +func (c *ServerConn) IsHerculesSupported() bool { + _, herculesSupported := c.features["HERCULES"] + return herculesSupported +} + +// Striped Passive +// +// This command is analogous to the PASV command, but allows an array of +// host/port connections to be returned. This enables STRIPING, that is, +// multiple network endpoints (multi-homed hosts, or multiple hosts) to +// participate in the transfer. +func (c *ServerConn) spas() ([]string, error) { + _, line, err := c.cmd(StatusExtendedPassiveMode, "SPAS") + if err != nil { + return nil, err + } + + lines := strings.Split(line, "\n") + + var addrs []string + + for _, line = range lines { + if !strings.HasPrefix(line, " ") { + continue + } + + addrs = append(addrs, strings.TrimLeft(line, " ")) + } + + return addrs, nil +} + +// Extended Retrieve +// +// This is analogous to the RETR command, but it allows the data to be +// manipulated (typically reduced in size) before being transmitted. +func (c *ServerConn) Eret(path string, offset, length int) (*Response, error) { + + conn, err := c.cmdDataConnFrom(0, "ERET PFT=\"%d,%d\" %s", offset, length, path) + + if err != nil { + return nil, err + } + + return &Response{conn: conn, c: c}, nil +} + +// Options to RETR +// +// The options described in this section provide a means to convey +// striping and transfer parallelism information to the server-DTP. +// For the RETR command, the Client-FTP may specify a parallelism and +// striping mode it wishes the server-DTP to use. These options are +// only used by the server-DTP if the retrieve operation is done in +// extended block mode. These options are implemented as RFC 2389 +// extensions. +func (c *ServerConn) SetRetrOpts(parallelism, blockSize int) error { + if parallelism < 1 { + return fmt.Errorf("parallelism needs to be at least 1") + } + + if blockSize < 1 { + return fmt.Errorf("block size needs to be at least 1") + } + + parallelOpts := "Parallelism=" + strconv.Itoa(parallelism) + ";" + layoutOpts := "StripeLayout=Blocked;BlockSize=" + strconv.Itoa(blockSize) + ";" + + code, message, err := c.cmd(-1, "OPTS RETR "+parallelOpts+layoutOpts) + if err != nil { + return err + } + + if code != StatusCommandOK { + return fmt.Errorf("failed to set options: %s", message) + } + + c.blockSize = blockSize + + return nil +} diff --git a/ftp/internal/ftp/conn_test.go b/ftp/internal/ftp/conn_test.go new file mode 100644 index 00000000..e228712b --- /dev/null +++ b/ftp/internal/ftp/conn_test.go @@ -0,0 +1,356 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +// +// Copyright 2019-2021 ETH Zurich modifications to add support for SCION + +package ftp + +import ( + "errors" + "io" + "io/ioutil" + "net" + "net/textproto" + "reflect" + "strconv" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" +) + +type ftpMock struct { + address string + listener *net.TCPListener + proto *textproto.Conn + commands []string // list of received commands + rest int + dataConn *mockDataConn + sync.WaitGroup +} + +// newFtpMock returns a mock implementation of a FTP server +// For simplication, a mock instance only accepts a signle connection and terminates afer +func newFtpMock(t *testing.T, address string) (*ftpMock, error) { + var err error + mock := &ftpMock{address: address} + + l, err := net.Listen("tcp", address+":0") + if err != nil { + return nil, err + } + + tcpListener, ok := l.(*net.TCPListener) + if !ok { + return nil, errors.New("listener is not a net.TCPListener") + } + mock.listener = tcpListener + + go mock.listen(t) + + return mock, nil +} + +func (mock *ftpMock) listen(t *testing.T) { + // Listen for an incoming connection. + conn, err := mock.listener.Accept() + assert.NoError(t, err) + + // Do not accept incoming connections anymore + err = mock.listener.Close() + assert.NoError(t, err) + + mock.Add(1) + defer mock.Done() + defer func() { assert.NoError(t, conn.Close()) }() + + mock.proto = textproto.NewConn(conn) + assert.NoError(t, mock.proto.Writer.PrintfLine("220 FTP Server ready.")) + + for { + fullCommand, _ := mock.proto.ReadLine() + + cmdParts := strings.Split(fullCommand, " ") + + // Append to list of received commands + mock.commands = append(mock.commands, cmdParts[0]) + + // At least one command must have a multiline response + switch cmdParts[0] { + case "FEAT": + assert.NoError(t, mock.proto.Writer.PrintfLine("211-Features:\r\n FEAT\r\n PASV\r\n EPSV\r\n SIZE\r\n211 End")) + case "USER": + if cmdParts[1] == "anonymous" { + assert.NoError(t, mock.proto.Writer.PrintfLine("331 Please send your password")) + } else { + assert.NoError(t, mock.proto.Writer.PrintfLine("530 This FTP server is anonymous only")) + } + case "PASS": + assert.NoError(t, mock.proto.Writer.PrintfLine("230-Hey,\r\nWelcome to my FTP\r\n230 Access granted")) + case "TYPE": + assert.NoError(t, mock.proto.Writer.PrintfLine("200 Type set ok")) + case "CWD": + if cmdParts[1] == "missing-dir" { + assert.NoError(t, mock.proto.Writer.PrintfLine("550 %s: No such file or directory", cmdParts[1])) + } else { + assert.NoError(t, mock.proto.Writer.PrintfLine("250 Directory successfully changed.")) + } + case "DELE": + assert.NoError(t, mock.proto.Writer.PrintfLine("250 File successfully removed.")) + case "MKD": + assert.NoError(t, mock.proto.Writer.PrintfLine("257 Directory successfully created.")) + case "RMD": + if cmdParts[1] == "missing-dir" { + assert.NoError(t, mock.proto.Writer.PrintfLine("550 No such file or directory")) + } else { + assert.NoError(t, mock.proto.Writer.PrintfLine("250 Directory successfully removed.")) + } + case "PWD": + assert.NoError(t, mock.proto.Writer.PrintfLine("257 \"/incoming\"")) + case "CDUP": + assert.NoError(t, mock.proto.Writer.PrintfLine("250 CDUP command successful")) + case "SIZE": + if cmdParts[1] == "magic-file" { + assert.NoError(t, mock.proto.Writer.PrintfLine("213 42")) + } else { + assert.NoError(t, mock.proto.Writer.PrintfLine("550 Could not get file size.")) + } + case "PASV": + p, err := mock.listenDataConn() + if err != nil { + assert.NoError(t, mock.proto.Writer.PrintfLine("451 %s.", err)) + break + } + + p1 := int(p / 256) + p2 := p % 256 + + assert.NoError(t, mock.proto.Writer.PrintfLine("227 Entering Passive Mode (127,0,0,1,%d,%d).", p1, p2)) + case "EPSV": + p, err := mock.listenDataConn() + if err != nil { + assert.NoError(t, mock.proto.Writer.PrintfLine("451 %s.", err)) + break + } + assert.NoError(t, mock.proto.Writer.PrintfLine("229 Entering Extended Passive Mode (|||%d|)", p)) + case "STOR": + if mock.dataConn == nil { + assert.NoError(t, mock.proto.Writer.PrintfLine("425 Unable to build data connection: Connection refused")) + break + } + assert.NoError(t, mock.proto.Writer.PrintfLine("150 please send")) + mock.recvDataConn() + case "LIST": + if mock.dataConn == nil { + assert.NoError(t, mock.proto.Writer.PrintfLine("425 Unable to build data connection: Connection refused")) + break + } + + mock.dataConn.Wait() + assert.NoError(t, mock.proto.Writer.PrintfLine("150 Opening ASCII mode data connection for file list")) + _, err = mock.dataConn.conn.Write([]byte("-rw-r--r-- 1 ftp wheel 0 Jan 29 10:29 lo")) + assert.NoError(t, err) + assert.NoError(t, mock.proto.Writer.PrintfLine("226 Transfer complete")) + _ = mock.closeDataConn() + case "NLST": + if mock.dataConn == nil { + assert.NoError(t, mock.proto.Writer.PrintfLine("425 Unable to build data connection: Connection refused")) + break + } + + mock.dataConn.Wait() + assert.NoError(t, mock.proto.Writer.PrintfLine("150 Opening ASCII mode data connection for file list")) + _, err = mock.dataConn.conn.Write([]byte("/incoming")) + assert.NoError(t, err) + assert.NoError(t, mock.proto.Writer.PrintfLine("226 Transfer complete")) + assert.NoError(t, mock.closeDataConn()) + case "RETR": + if mock.dataConn == nil { + assert.NoError(t, mock.proto.Writer.PrintfLine("425 Unable to build data connection: Connection refused")) + break + } + + mock.dataConn.Wait() + assert.NoError(t, mock.proto.Writer.PrintfLine("150 Opening ASCII mode data connection for file list")) + _, err = mock.dataConn.conn.Write([]byte(testData[mock.rest:])) + assert.NoError(t, err) + mock.rest = 0 + assert.NoError(t, mock.proto.Writer.PrintfLine("226 Transfer complete")) + assert.NoError(t, mock.closeDataConn()) + case "RNFR": + assert.NoError(t, mock.proto.Writer.PrintfLine("350 File or directory exists, ready for destination name")) + case "RNTO": + assert.NoError(t, mock.proto.Writer.PrintfLine("250 Rename successful")) + case "REST": + if len(cmdParts) != 2 { + assert.NoError(t, mock.proto.Writer.PrintfLine("500 wrong number of arguments")) + break + } + rest, err := strconv.Atoi(cmdParts[1]) + if err != nil { + assert.NoError(t, mock.proto.Writer.PrintfLine("500 REST: %s", err)) + break + } + mock.rest = rest + assert.NoError(t, mock.proto.Writer.PrintfLine("350 Restarting at %s. Send STORE or RETRIEVE to initiate transfer", cmdParts[1])) + case "NOOP": + _ = mock.proto.Writer.PrintfLine("200 NOOP ok.") + case "REIN": + assert.NoError(t, mock.proto.Writer.PrintfLine("220 Logged out")) + case "QUIT": + assert.NoError(t, mock.proto.Writer.PrintfLine("221 Goodbye.")) + return + default: + assert.NoError(t, mock.proto.Writer.PrintfLine("500 Unknown command %s.", cmdParts[0])) + } + } +} + +func (mock *ftpMock) closeDataConn() (err error) { + if mock.dataConn != nil { + err = mock.dataConn.Close() + mock.dataConn = nil + } + return +} + +type mockDataConn struct { + listener *net.TCPListener + conn net.Conn + // WaitGroup is done when conn is accepted and stored + sync.WaitGroup +} + +func (d *mockDataConn) Close() (err error) { + if d.listener != nil { + err = d.listener.Close() + } + if d.conn != nil { + err = d.conn.Close() + } + return +} + +func (mock *ftpMock) listenDataConn() (int64, error) { + if err := mock.closeDataConn(); err != nil { + return 0, err + } + + l, err := net.Listen("tcp", mock.address+":0") + if err != nil { + return 0, err + } + + tcpListener, ok := l.(*net.TCPListener) + if !ok { + return 0, errors.New("listener is not a net.TCPListener") + } + + addr := tcpListener.Addr().String() + + _, port, err := net.SplitHostPort(addr) + if err != nil { + return 0, err + } + + p, err := strconv.ParseInt(port, 10, 32) + if err != nil { + return 0, err + } + + dataConn := &mockDataConn{listener: tcpListener} + dataConn.Add(1) + + go func() { + // Listen for an incoming connection. + conn, err := dataConn.listener.Accept() + if err != nil { + // t.Errorf("can not accept: %s", err) + return + } + + dataConn.conn = conn + dataConn.Done() + }() + + mock.dataConn = dataConn + return p, nil +} + +func (mock *ftpMock) recvDataConn() { + mock.dataConn.Wait() + _, _ = io.Copy(ioutil.Discard, mock.dataConn.conn) + _ = mock.proto.Writer.PrintfLine("226 Transfer Complete") + _ = mock.closeDataConn() +} + +func (mock *ftpMock) Addr() string { + return mock.listener.Addr().String() +} + +// Closes the listening socket +func (mock *ftpMock) Close() { + _ = mock.listener.Close() +} + +// Helper to return a scionftp connected to a mock server +func openConn(t *testing.T, addr string, options ...DialOption) (*ftpMock, *ServerConn) { + mock, err := newFtpMock(t, addr) + if err != nil { + t.Fatal(err) + } + defer mock.Close() + + c, err := Dial(addr, options...) + if err != nil { + t.Fatal(err) + } + + err = c.Login("anonymous", "anonymous") + if err != nil { + t.Fatal(err) + } + + return mock, c + +} + +// Helper to close a scionftp connected to a mock server +func closeConn(t *testing.T, mock *ftpMock, c *ServerConn, commands []string) { + expected := []string{"FEAT", "USER", "PASS", "TYPE"} + expected = append(expected, commands...) + expected = append(expected, "QUIT") + + if err := c.Quit(); err != nil { + t.Fatal(err) + } + + // Wait for the connection to close + mock.Wait() + + if !reflect.DeepEqual(mock.commands, expected) { + t.Fatal("unexpected sequence of commands:", mock.commands, "expected:", expected) + } +} + +func TestConn4(t *testing.T) { + mock, c := openConn(t, "127.0.0.1") + closeConn(t, mock, c, nil) +} + +func TestConn6(t *testing.T) { + mock, c := openConn(t, "[::1]") + closeConn(t, mock, c, nil) +} diff --git a/ftp/internal/ftp/debug.go b/ftp/internal/ftp/debug.go new file mode 100644 index 00000000..e348a24c --- /dev/null +++ b/ftp/internal/ftp/debug.go @@ -0,0 +1,35 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +package ftp + +import "io" + +type debugWrapper struct { + conn io.ReadWriteCloser + io.Reader + io.Writer +} + +func newDebugWrapper(conn io.ReadWriteCloser, w io.Writer) io.ReadWriteCloser { + return &debugWrapper{ + Reader: io.TeeReader(conn, w), + Writer: io.MultiWriter(w, conn), + conn: conn, + } +} + +func (w *debugWrapper) Close() error { + return w.conn.Close() +} diff --git a/ftp/internal/ftp/ftp.go b/ftp/internal/ftp/ftp.go new file mode 100644 index 00000000..7ee93c96 --- /dev/null +++ b/ftp/internal/ftp/ftp.go @@ -0,0 +1,193 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +// +// Copyright 2020-2021 ETH Zurich modifications to add support for SCION + +// Package ftp implements an FTP client as described in RFC 959. Non-standard +// modifications were made to support SCION, Hercules mode and the GridFTP +// extension. +// +// A textproto.Error is returned for errors at the protocol level. +package ftp + +import ( + "context" + "fmt" + "io" + "net" + "net/textproto" + "time" + + "github.com/netsec-ethz/scion-apps/internal/ftp/socket" +) + +// EntryType describes the different types of an Entry. +type EntryType int + +// The differents types of an Entry +const ( + EntryTypeFile EntryType = iota + EntryTypeFolder + EntryTypeLink +) + +// ServerConn represents the connection to a remote FTP server. +// A single connection only supports one in-flight data connection. +// It is not safe to be called concurrently. +type ServerConn struct { + options *dialOptions + socket *socket.SingleStream + conn *textproto.Conn + features map[string]string // Server capabilities discovered at runtime + mode byte + blockSize int +} + +// DialOption represents an option to start a new connection with Dial +type DialOption struct { + setup func(do *dialOptions) +} + +// dialOptions contains all the options set by DialOption.setup +type dialOptions struct { + context context.Context + dialer net.Dialer + disableEPSV bool + location *time.Location + debugOutput io.Writer + blockSize int +} + +// Entry describes a file and is returned by List(). +type Entry struct { + Name string + Type EntryType + Size uint64 + Time time.Time +} + +// Dial connects to the specified address with optional options +func Dial(remote string, options ...DialOption) (*ServerConn, error) { + do := &dialOptions{} + for _, option := range options { + option.setup(do) + } + + if do.location == nil { + do.location = time.UTC + } + + maxChunkSize := do.blockSize + if maxChunkSize == 0 { + maxChunkSize = 500 + } + + conn, err := socket.DialAddr(remote) + if err != nil { + return nil, err + } + + var sourceConn io.ReadWriteCloser = conn + if do.debugOutput != nil { + sourceConn = newDebugWrapper(conn, do.debugOutput) + } + + c := &ServerConn{ + options: do, + features: make(map[string]string), + socket: conn, + conn: textproto.NewConn(sourceConn), + blockSize: maxChunkSize, + } + + _, _, err = c.conn.ReadResponse(StatusReady) + if err != nil { + if err2 := c.Quit(); err2 != nil { + return nil, fmt.Errorf("could not read response: %s\nand could not close connection: %s", err, err2) + } + return nil, fmt.Errorf("could not read response: %s", err) + } + + err = c.feat() + if err != nil { + if err2 := c.Quit(); err2 != nil { + return nil, fmt.Errorf("could execute FEAT: %s\nand could not close connection: %s", err, err2) + } + return nil, fmt.Errorf("could execute FEAT: %s", err) + } + + return c, nil +} + +// DialWithTimeout returns a DialOption that configures the ServerConn with specified timeout +func DialWithTimeout(timeout time.Duration) DialOption { + return DialOption{func(do *dialOptions) { + do.dialer.Timeout = timeout + }} +} + +// DialWithDialer returns a DialOption that configures the ServerConn with specified net.Dialer +func DialWithDialer(dialer net.Dialer) DialOption { + return DialOption{func(do *dialOptions) { + do.dialer = dialer + }} +} + +// DialWithDisabledEPSV returns a DialOption that configures the ServerConn with EPSV disabled +// Note that EPSV is only used when advertised in the server features. +func DialWithDisabledEPSV(disabled bool) DialOption { + return DialOption{func(do *dialOptions) { + do.disableEPSV = disabled + }} +} + +// DialWithLocation returns a DialOption that configures the ServerConn with specified time.Location +// The location is used to parse the dates sent by the server which are in server's timezone +func DialWithLocation(location *time.Location) DialOption { + return DialOption{func(do *dialOptions) { + do.location = location + }} +} + +// DialWithContext returns a DialOption that configures the ServerConn with specified context +// The context will be used for the initial connection setup +func DialWithContext(ctx context.Context) DialOption { + return DialOption{func(do *dialOptions) { + do.context = ctx + }} +} + +// DialWithDebugOutput returns a DialOption that configures the ServerConn to write to the Writer +// everything it reads from the server +func DialWithDebugOutput(w io.Writer) DialOption { + return DialOption{func(do *dialOptions) { + do.debugOutput = w + }} +} + +// DialWithBlockSize sets the maximum blocksize to be used at the start but only clientside, +// alternatively we can set it with the command OPTS RETR (SetRetrOpts) +func DialWithBlockSize(blockSize int) DialOption { + return DialOption{func(do *dialOptions) { + do.blockSize = blockSize + }} +} + +// DialTimeout initializes the connection to the specified ftp server address. +// +// It is generally followed by a call to Login() as most FTP commands require +// an authenticated user. +func DialTimeout(remote string, timeout time.Duration) (*ServerConn, error) { + return Dial(remote, DialWithTimeout(timeout)) +} diff --git a/ftp/internal/ftp/parse.go b/ftp/internal/ftp/parse.go new file mode 100644 index 00000000..36065093 --- /dev/null +++ b/ftp/internal/ftp/parse.go @@ -0,0 +1,275 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +// +// Copyright 2020-2021 ETH Zurich modifications to add support for SCION + +package ftp + +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +var errUnsupportedListLine = errors.New("unsupported LIST line") +var errUnsupportedListDate = errors.New("unsupported LIST date") +var errUnknownListEntryType = errors.New("unknown entry type") + +type parseFunc func(string, time.Time, *time.Location) (*Entry, error) + +var listLineParsers = []parseFunc{ + parseRFC3659ListLine, + parseLsListLine, + parseDirListLine, + parseHostedFTPLine, +} + +var dirTimeFormats = []string{ + "01-02-06 03:04PM", + "2006-01-02 15:04", +} + +// parseRFC3659ListLine parses the style of directory line defined in RFC 3659. +func parseRFC3659ListLine(line string, now time.Time, loc *time.Location) (*Entry, error) { + iSemicolon := strings.Index(line, ";") + iWhitespace := strings.Index(line, " ") + + if iSemicolon < 0 || iSemicolon > iWhitespace { + return nil, errUnsupportedListLine + } + + e := &Entry{ + Name: line[iWhitespace+1:], + } + + for _, field := range strings.Split(line[:iWhitespace-1], ";") { + i := strings.Index(field, "=") + if i < 1 { + return nil, errUnsupportedListLine + } + + key := strings.ToLower(field[:i]) + value := field[i+1:] + + switch key { + case "modify": + var err error + e.Time, err = time.ParseInLocation("20060102150405", value, loc) + if err != nil { + return nil, err + } + case "type": + switch value { + case "dir", "cdir", "pdir": + e.Type = EntryTypeFolder + case "file": + e.Type = EntryTypeFile + } + case "size": + if err := e.setSize(value); err != nil { + return nil, err + } + } + } + return e, nil +} + +// parseLsListLine parses a directory line in a format based on the output of +// the UNIX ls command. +func parseLsListLine(line string, now time.Time, loc *time.Location) (*Entry, error) { + + // Has the first field a length of 10 bytes? + if strings.IndexByte(line, ' ') != 10 { + return nil, errUnsupportedListLine + } + + scanner := newScanner(line) + fields := scanner.NextFields(6) + + if len(fields) < 6 { + return nil, errUnsupportedListLine + } + + if fields[1] == "folder" && fields[2] == "0" { + e := &Entry{ + Type: EntryTypeFolder, + Name: scanner.Remaining(), + } + if err := e.setTime(fields[3:6], now, loc); err != nil { + return nil, err + } + + return e, nil + } + + if fields[1] == "0" { + fields = append(fields, scanner.Next()) + e := &Entry{ + Type: EntryTypeFile, + Name: scanner.Remaining(), + } + + if err := e.setSize(fields[2]); err != nil { + return nil, errUnsupportedListLine + } + if err := e.setTime(fields[4:7], now, loc); err != nil { + return nil, err + } + + return e, nil + } + + // Read two more fields + fields = append(fields, scanner.NextFields(2)...) + if len(fields) < 8 { + return nil, errUnsupportedListLine + } + + e := &Entry{ + Name: scanner.Remaining(), + } + switch fields[0][0] { + case '-': + e.Type = EntryTypeFile + if err := e.setSize(fields[4]); err != nil { + return nil, err + } + case 'd': + e.Type = EntryTypeFolder + case 'l': + e.Type = EntryTypeLink + default: + return nil, errUnknownListEntryType + } + + if err := e.setTime(fields[5:8], now, loc); err != nil { + return nil, err + } + + return e, nil +} + +// parseDirListLine parses a directory line in a format based on the output of +// the MS-DOS DIR command. +func parseDirListLine(line string, now time.Time, loc *time.Location) (*Entry, error) { + e := &Entry{} + var err error + + // Try various time formats that DIR might use, and stop when one works. + for _, format := range dirTimeFormats { + if len(line) > len(format) { + e.Time, err = time.ParseInLocation(format, line[:len(format)], loc) + if err == nil { + line = line[len(format):] + break + } + } + } + if err != nil { + // None of the time formats worked. + return nil, errUnsupportedListLine + } + + line = strings.TrimLeft(line, " ") + if strings.HasPrefix(line, "") { + e.Type = EntryTypeFolder + line = strings.TrimPrefix(line, "") + } else { + space := strings.Index(line, " ") + if space == -1 { + return nil, errUnsupportedListLine + } + e.Size, err = strconv.ParseUint(line[:space], 10, 64) + if err != nil { + return nil, errUnsupportedListLine + } + e.Type = EntryTypeFile + line = line[space:] + } + + e.Name = strings.TrimLeft(line, " ") + return e, nil +} + +// parseHostedFTPLine parses a directory line in the non-standard format used +// by hostedftp.com +// -r-------- 0 user group 65222236 Feb 24 00:39 UABlacklistingWeek8.csv +// (The link count is inexplicably 0) +func parseHostedFTPLine(line string, now time.Time, loc *time.Location) (*Entry, error) { + // Has the first field a length of 10 bytes? + if strings.IndexByte(line, ' ') != 10 { + return nil, errUnsupportedListLine + } + + scanner := newScanner(line) + fields := scanner.NextFields(2) + + if len(fields) < 2 || fields[1] != "0" { + return nil, errUnsupportedListLine + } + + // Set link count to 1 and attempt to parse as Unix. + return parseLsListLine(fields[0]+" 1 "+scanner.Remaining(), now, loc) +} + +// parseListLine parses the various non-standard format returned by the LIST +// FTP command. +func parseListLine(line string, now time.Time, loc *time.Location) (*Entry, error) { + for _, f := range listLineParsers { + e, err := f(line, now, loc) + if err != errUnsupportedListLine { + return e, err + } + } + return nil, errUnsupportedListLine +} + +func (e *Entry) setSize(str string) (err error) { + e.Size, err = strconv.ParseUint(str, 0, 64) + return +} + +func (e *Entry) setTime(fields []string, now time.Time, loc *time.Location) (err error) { + if strings.Contains(fields[2], ":") { // contains time + thisYear, _, _ := now.Date() + timeStr := fmt.Sprintf("%s %s %d %s", fields[1], fields[0], thisYear, fields[2]) + e.Time, err = time.ParseInLocation("_2 Jan 2006 15:04", timeStr, loc) + + /* + On unix, `info ls` shows: + + 10.1.6 Formatting file timestamps + --------------------------------- + + A timestamp is considered to be “recent” if it is less than six + months old, and is not dated in the future. If a timestamp dated today + is not listed in recent form, the timestamp is in the future, which + means you probably have clock skew problems which may break programs + like ‘make’ that rely on file timestamps. + */ + if !e.Time.Before(now.AddDate(0, 6, 0)) { + e.Time = e.Time.AddDate(-1, 0, 0) + } + + } else { // only the date + if len(fields[2]) != 4 { + return errUnsupportedListDate + } + timeStr := fmt.Sprintf("%s %s %s 00:00", fields[1], fields[0], fields[2]) + e.Time, err = time.ParseInLocation("_2 Jan 2006 15:04", timeStr, loc) + } + return +} diff --git a/ftp/internal/ftp/parse_test.go b/ftp/internal/ftp/parse_test.go new file mode 100644 index 00000000..7d2abc37 --- /dev/null +++ b/ftp/internal/ftp/parse_test.go @@ -0,0 +1,184 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +// +// Copyright 2020-2021 ETH Zurich modifications to add support for SCION + +package ftp + +import ( + "strings" + "testing" + "time" +) + +var ( + // now is the current time for all tests + now = newTime(2017, time.March, 10, 23, 00) + + thisYear, _, _ = now.Date() + previousYear = thisYear - 1 +) + +type line struct { + line string + name string + size uint64 + entryType EntryType + time time.Time +} + +type unsupportedLine struct { + line string + err error +} + +var listTests = []line{ + // UNIX ls -l style + {"drwxr-xr-x 3 110 1002 3 Dec 02 2009 pub", "pub", 0, EntryTypeFolder, newTime(2009, time.December, 2)}, + {"drwxr-xr-x 3 110 1002 3 Dec 02 2009 p u b", "p u b", 0, EntryTypeFolder, newTime(2009, time.December, 2)}, + {"-rw-r--r-- 1 marketwired marketwired 12016 Mar 16 2016 2016031611G087802-001.newsml", "2016031611G087802-001.newsml", 12016, EntryTypeFile, newTime(2016, time.March, 16)}, + + {"-rwxr-xr-x 3 110 1002 1234567 Dec 02 2009 fileName", "fileName", 1234567, EntryTypeFile, newTime(2009, time.December, 2)}, + {"lrwxrwxrwx 1 root other 7 Jan 25 00:17 bin -> usr/bin", "bin -> usr/bin", 0, EntryTypeLink, newTime(thisYear, time.January, 25, 0, 17)}, + + // Another ls style + {"drwxr-xr-x folder 0 Aug 15 05:49 !!!-Tipp des Haus!", "!!!-Tipp des Haus!", 0, EntryTypeFolder, newTime(thisYear, time.August, 15, 5, 49)}, + {"drwxrwxrwx folder 0 Aug 11 20:32 P0RN", "P0RN", 0, EntryTypeFolder, newTime(thisYear, time.August, 11, 20, 32)}, + {"-rw-r--r-- 0 18446744073709551615 18446744073709551615 Nov 16 2006 VIDEO_TS.VOB", "VIDEO_TS.VOB", 18446744073709551615, EntryTypeFile, newTime(2006, time.November, 16)}, + + // Microsoft's FTP servers for Windows + {"---------- 1 owner group 1803128 Jul 10 10:18 ls-lR.Z", "ls-lR.Z", 1803128, EntryTypeFile, newTime(thisYear, time.July, 10, 10, 18)}, + {"d--------- 1 owner group 0 Nov 9 19:45 Softlib", "Softlib", 0, EntryTypeFolder, newTime(previousYear, time.November, 9, 19, 45)}, + + // WFTPD for MSDOS + {"-rwxrwxrwx 1 noone nogroup 322 Aug 19 1996 message.ftp", "message.ftp", 322, EntryTypeFile, newTime(1996, time.August, 19)}, + + // RFC3659 format: https://tools.ietf.org/html/rfc3659#section-7 + {"modify=20150813224845;perm=fle;type=cdir;unique=119FBB87U4;UNIX.group=0;UNIX.mode=0755;UNIX.owner=0; .", ".", 0, EntryTypeFolder, newTime(2015, time.August, 13, 22, 48, 45)}, + {"modify=20150813224845;perm=fle;type=pdir;unique=119FBB87U4;UNIX.group=0;UNIX.mode=0755;UNIX.owner=0; ..", "..", 0, EntryTypeFolder, newTime(2015, time.August, 13, 22, 48, 45)}, + {"modify=20150806235817;perm=fle;type=dir;unique=1B20F360U4;UNIX.group=0;UNIX.mode=0755;UNIX.owner=0; movies", "movies", 0, EntryTypeFolder, newTime(2015, time.August, 6, 23, 58, 17)}, + {"modify=20150814172949;perm=flcdmpe;type=dir;unique=85A0C168U4;UNIX.group=0;UNIX.mode=0777;UNIX.owner=0; _upload", "_upload", 0, EntryTypeFolder, newTime(2015, time.August, 14, 17, 29, 49)}, + {"modify=20150813175250;perm=adfr;size=951;type=file;unique=119FBB87UE;UNIX.group=0;UNIX.mode=0644;UNIX.owner=0; welcome.msg", "welcome.msg", 951, EntryTypeFile, newTime(2015, time.August, 13, 17, 52, 50)}, + // Format and types have first letter UpperCase + {"Modify=20150813175250;Perm=adfr;Size=951;Type=file;Unique=119FBB87UE;UNIX.group=0;UNIX.mode=0644;UNIX.owner=0; welcome.msg", "welcome.msg", 951, EntryTypeFile, newTime(2015, time.August, 13, 17, 52, 50)}, + + // DOS DIR command output + {"08-07-15 07:50PM 718 Post_PRR_20150901_1166_265118_13049.dat", "Post_PRR_20150901_1166_265118_13049.dat", 718, EntryTypeFile, newTime(2015, time.August, 7, 19, 50)}, + {"08-10-15 02:04PM Billing", "Billing", 0, EntryTypeFolder, newTime(2015, time.August, 10, 14, 4)}, + + // dir and file names that contain multiple spaces + {"drwxr-xr-x 3 110 1002 3 Dec 02 2009 spaces dir name", "spaces dir name", 0, EntryTypeFolder, newTime(2009, time.December, 2)}, + {"-rwxr-xr-x 3 110 1002 1234567 Dec 02 2009 file name", "file name", 1234567, EntryTypeFile, newTime(2009, time.December, 2)}, + {"-rwxr-xr-x 3 110 1002 1234567 Dec 02 2009 foo bar ", " foo bar ", 1234567, EntryTypeFile, newTime(2009, time.December, 2)}, + + // Odd link count from hostedftp.com + {"-r-------- 0 user group 65222236 Feb 24 00:39 RegularFile", "RegularFile", 65222236, EntryTypeFile, newTime(thisYear, time.February, 24, 0, 39)}, +} + +// Not supported, we expect a specific error message +var listTestsFail = []unsupportedLine{ + {"d [R----F--] supervisor 512 Jan 16 18:53 login", errUnsupportedListLine}, + {"- [R----F--] rhesus 214059 Oct 20 15:27 cx.exe", errUnsupportedListLine}, + {"drwxr-xr-x 3 110 1002 3 Dec 02 209 pub", errUnsupportedListDate}, + {"modify=20150806235817;invalid;UNIX.owner=0; movies", errUnsupportedListLine}, + {"Zrwxrwxrwx 1 root other 7 Jan 25 00:17 bin -> usr/bin", errUnknownListEntryType}, + {"total 1", errUnsupportedListLine}, + {"000000000x ", errUnsupportedListLine}, // see https://github.com/jlaffaye/ftp/issues/97 + {"", errUnsupportedListLine}, +} + +func TestParseValidListLine(t *testing.T) { + for _, lt := range listTests { + entry, err := parseListLine(lt.line, now, time.UTC) + if err != nil { + t.Errorf("parseListLine(%v) returned err = %v", lt.line, err) + continue + } + if entry.Name != lt.name { + t.Errorf("parseListLine(%v).Name = '%v', want '%v'", lt.line, entry.Name, lt.name) + } + if entry.Type != lt.entryType { + t.Errorf("parseListLine(%v).EntryType = %v, want %v", lt.line, entry.Type, lt.entryType) + } + if entry.Size != lt.size { + t.Errorf("parseListLine(%v).Size = %v, want %v", lt.line, entry.Size, lt.size) + } + if !entry.Time.Equal(lt.time) { + t.Errorf("parseListLine(%v).Time = %v, want %v", lt.line, entry.Time, lt.time) + } + } +} + +func TestParseUnsupportedListLine(t *testing.T) { + for _, lt := range listTestsFail { + _, err := parseListLine(lt.line, now, time.UTC) + if err == nil { + t.Errorf("parseListLine(%v) expected to fail", lt.line) + } else if err != lt.err { + t.Errorf("parseListLine(%v) expected to fail with error: '%s'; was: '%s'", lt.line, lt.err.Error(), err.Error()) + } + } +} + +func TestSettime(t *testing.T) { + tests := []struct { + line string + expected time.Time + }{ + // this year, in the past + {"Feb 10 23:00", newTime(thisYear, time.February, 10, 23)}, + + // this year, less than six months in the future + {"Sep 10 22:59", newTime(thisYear, time.September, 10, 22, 59)}, + + // previous year, otherwise it would be more than 6 months in the future + {"Sep 10 23:00", newTime(previousYear, time.September, 10, 23)}, + + // far in the future + {"Jan 23 2019", newTime(2019, time.January, 23)}, + } + + for _, test := range tests { + entry := &Entry{} + if err := entry.setTime(strings.Fields(test.line), now, time.UTC); err != nil { + t.Error(err) + } + + if !entry.Time.Equal(test.expected) { + t.Errorf("setTime(%v).Time = %v, want %v", test.line, entry.Time, test.expected) + } + } +} + +// newTime builds a UTC time from the given year, month, day, hour and minute +func newTime(year int, month time.Month, day int, hourMinSec ...int) time.Time { + var hour, min, sec int + + switch len(hourMinSec) { + case 0: + // nothing + case 3: + sec = hourMinSec[2] + fallthrough + case 2: + min = hourMinSec[1] + fallthrough + case 1: + hour = hourMinSec[0] + default: + panic("too many arguments") + } + + return time.Date(year, month, day, hour, min, sec, 0, time.UTC) +} diff --git a/ftp/internal/ftp/response.go b/ftp/internal/ftp/response.go new file mode 100644 index 00000000..d966c47f --- /dev/null +++ b/ftp/internal/ftp/response.go @@ -0,0 +1,54 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +// +// Copyright 2019-2021 ETH Zurich modifications to add support for SCION + +package ftp + +import ( + "net" + "time" +) + +// Response represents a data-connection +type Response struct { + conn net.Conn + c *ServerConn + closed bool +} + +// Read implements the io.Reader interface on a FTP data connection. +func (r *Response) Read(buf []byte) (int, error) { + return r.conn.Read(buf) +} + +// Close implements the io.Closer interface on a FTP data connection. +// After the first call, Close will do nothing and return nil. +func (r *Response) Close() error { + if r.closed { + return nil + } + err := r.conn.Close() + _, _, err2 := r.c.conn.ReadResponse(StatusClosingDataConnection) + if err2 != nil { + err = err2 + } + r.closed = true + return err +} + +// SetDeadline sets the deadlines associated with the connection. +func (r *Response) SetDeadline(t time.Time) error { + return r.conn.SetDeadline(t) +} diff --git a/ftp/internal/ftp/scanner.go b/ftp/internal/ftp/scanner.go new file mode 100644 index 00000000..78cc23be --- /dev/null +++ b/ftp/internal/ftp/scanner.go @@ -0,0 +1,74 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +// +// Copyright 2019-2021 ETH Zurich modifications to add support for SCION + +package ftp + +// A scanner for fields delimited by one or more whitespace characters +type scanner struct { + bytes []byte + position int +} + +// newScanner creates a new scanner +func newScanner(str string) *scanner { + return &scanner{ + bytes: []byte(str), + } +} + +// NextFields returns the next `count` fields +func (s *scanner) NextFields(count int) []string { + fields := make([]string, 0, count) + for i := 0; i < count; i++ { + if field := s.Next(); field != "" { + fields = append(fields, field) + } else { + break + } + } + return fields +} + +// Next returns the next field +func (s *scanner) Next() string { + sLen := len(s.bytes) + + // skip trailing whitespace + for s.position < sLen { + if s.bytes[s.position] != ' ' { + break + } + s.position++ + } + + start := s.position + + // skip non-whitespace + for s.position < sLen { + if s.bytes[s.position] == ' ' { + s.position++ + return string(s.bytes[start : s.position-1]) + } + s.position++ + } + + return string(s.bytes[start:s.position]) +} + +// Remaining returns the remaining string +func (s *scanner) Remaining() string { + return string(s.bytes[s.position:len(s.bytes)]) +} diff --git a/ftp/internal/ftp/scanner_test.go b/ftp/internal/ftp/scanner_test.go new file mode 100644 index 00000000..caf7d6fa --- /dev/null +++ b/ftp/internal/ftp/scanner_test.go @@ -0,0 +1,44 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +// +// Copyright 2019-2021 ETH Zurich modifications to add support for SCION + +package ftp + +import "testing" +import "github.com/stretchr/testify/assert" + +func TestScanner(t *testing.T) { + assert := assert.New(t) + + s := newScanner("foo bar x y") + assert.Equal("foo", s.Next()) + assert.Equal(" bar x y", s.Remaining()) + assert.Equal("bar", s.Next()) + assert.Equal("x y", s.Remaining()) + assert.Equal("x", s.Next()) + assert.Equal(" y", s.Remaining()) + assert.Equal("y", s.Next()) + assert.Equal("", s.Next()) + assert.Equal("", s.Remaining()) +} + +func TestScannerEmpty(t *testing.T) { + assert := assert.New(t) + + s := newScanner("") + assert.Equal("", s.Next()) + assert.Equal("", s.Next()) + assert.Equal("", s.Remaining()) +} diff --git a/ftp/internal/ftp/status.go b/ftp/internal/ftp/status.go new file mode 100644 index 00000000..df7daac7 --- /dev/null +++ b/ftp/internal/ftp/status.go @@ -0,0 +1,125 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +package ftp + +// FTP status codes, defined in RFC 959 +const ( + StatusInitiating = 100 + StatusRestartMarker = 110 + StatusReadyMinute = 120 + StatusAlreadyOpen = 125 + StatusAboutToSend = 150 + + StatusCommandOK = 200 + StatusCommandNotImplemented = 202 + StatusSystem = 211 + StatusDirectory = 212 + StatusFile = 213 + StatusHelp = 214 + StatusName = 215 + StatusReady = 220 + StatusClosing = 221 + StatusDataConnectionOpen = 225 + StatusClosingDataConnection = 226 + StatusPassiveMode = 227 + StatusLongPassiveMode = 228 + StatusExtendedPassiveMode = 229 + StatusLoggedIn = 230 + StatusLoggedOut = 231 + StatusLogoutAck = 232 + StatusRequestedFileActionOK = 250 + StatusPathCreated = 257 + + StatusUserOK = 331 + StatusLoginNeedAccount = 332 + StatusRequestFilePending = 350 + + StatusNotAvailable = 421 + StatusCanNotOpenDataConnection = 425 + StatusTransfertAborted = 426 + StatusInvalidCredentials = 430 + StatusHostUnavailable = 434 + StatusFileActionIgnored = 450 + StatusActionAborted = 451 + Status452 = 452 + + StatusBadCommand = 500 + StatusBadArguments = 501 + StatusNotImplemented = 502 + StatusBadSequence = 503 + StatusNotImplementedParameter = 504 + StatusNotLoggedIn = 530 + StatusStorNeedAccount = 532 + StatusFileUnavailable = 550 + StatusPageTypeUnknown = 551 + StatusExceededStorage = 552 + StatusBadFileName = 553 +) + +var statusText = map[int]string{ + // 200 + StatusCommandOK: "Command okay.", + StatusCommandNotImplemented: "Command not implemented, superfluous at this site.", + StatusSystem: "System status, or system help reply.", + StatusDirectory: "Directory status.", + StatusFile: "File status.", + StatusHelp: "Help message.", + StatusName: "", + StatusReady: "Service ready for new user.", + StatusClosing: "Service closing control connection.", + StatusDataConnectionOpen: "Data connection open; no transfer in progress.", + StatusClosingDataConnection: "Closing data connection. Requested file action successful.", + StatusPassiveMode: "Entering Passive Mode.", + StatusLongPassiveMode: "Entering Long Passive Mode.", + StatusExtendedPassiveMode: "Entering Extended Passive Mode.", + StatusLoggedIn: "User logged in, proceed.", + StatusLoggedOut: "User logged out; service terminated.", + StatusLogoutAck: "Logout command noted, will complete when transfer done.", + StatusRequestedFileActionOK: "Requested file action okay, completed.", + StatusPathCreated: "Path created.", + + // 300 + StatusUserOK: "User name okay, need password.", + StatusLoginNeedAccount: "Need account for login.", + StatusRequestFilePending: "Requested file action pending further information.", + + // 400 + StatusNotAvailable: "Service not available, closing control connection.", + StatusCanNotOpenDataConnection: "Can't open data connection.", + StatusTransfertAborted: "Connection closed; transfer aborted.", + StatusInvalidCredentials: "Invalid username or password.", + StatusHostUnavailable: "Requested host unavailable.", + StatusFileActionIgnored: "Requested file action not taken.", + StatusActionAborted: "Requested action aborted. Local error in processing.", + Status452: "Insufficient storage space in system.", + + // 500 + StatusBadCommand: "Command unrecognized.", + StatusBadArguments: "Syntax error in parameters or arguments.", + StatusNotImplemented: "Command not implemented.", + StatusBadSequence: "Bad sequence of commands.", + StatusNotImplementedParameter: "Command not implemented for that parameter.", + StatusNotLoggedIn: "Not logged in.", + StatusStorNeedAccount: "Need account for storing files.", + StatusFileUnavailable: "File unavailable.", + StatusPageTypeUnknown: "Page type unknown.", + StatusExceededStorage: "Exceeded storage allocation.", + StatusBadFileName: "File name not allowed.", +} + +// StatusText returns a text for the FTP status code. It returns the empty string if the code is unknown. +func StatusText(code int) string { + return statusText[code] +} diff --git a/ftp/internal/ftp/status_test.go b/ftp/internal/ftp/status_test.go new file mode 100644 index 00000000..190c6262 --- /dev/null +++ b/ftp/internal/ftp/status_test.go @@ -0,0 +1,31 @@ +// Copyright (c) 2011-2013, Julien Laffaye +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +package ftp + +import "testing" + +func TestValidStatusText(t *testing.T) { + txt := StatusText(StatusInvalidCredentials) + if txt == "" { + t.Fatal("exptected status text, got empty string") + } +} + +func TestInvalidStatusText(t *testing.T) { + txt := StatusText(0) + if txt != "" { + t.Fatalf("got status text %q, expected empty string", txt) + } +} diff --git a/ftp/main.go b/ftp/main.go new file mode 100644 index 00000000..e2924c12 --- /dev/null +++ b/ftp/main.go @@ -0,0 +1,305 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "context" + "flag" + "fmt" + "io" + "os" + "strconv" + "strings" + + "github.com/netsec-ethz/scion-apps/ftp/internal/ftp" +) + +func main() { + app := App{ + ctx: context.Background(), + herculesBinary: *herculesFlag, + } + + app.cmd = commandMap{ + "help": app.help, + "connect": app.connect, + "login": app.login, + "ls": app.ls, + "cd": app.cd, + "pwd": app.pwd, + "mode": app.mode, + "get": app.retr, + "put": app.stor, + "mkdir": app.mkdir, + "quit": app.quit, + } + + if err := app.run(); err != nil { + fmt.Println(err) + } +} + +type commandMap map[string]func([]string) + +var ( + herculesFlag = flag.String("hercules", "", "Enable Hercules mode using the Hercules binary specified\nIn Hercules mode, scionFTP checks the following directories for Hercules config files: ., /etc, /etc/scion-ftp") +) + +func init() { + flag.Parse() +} + +type App struct { + conn *ftp.ServerConn + out io.Writer + cmd commandMap + ctx context.Context + herculesBinary string +} + +func (app *App) print(a interface{}) { + fmt.Fprintln(app.out, a) +} + +func (app *App) run() error { + scanner := bufio.NewReader(os.Stdin) + app.out = os.Stdout + + for { + fmt.Printf("> ") + input, err := scanner.ReadString('\n') + if err != nil { + return err + } + + args := strings.Split(strings.TrimSpace(input), " ") + if f, ok := app.cmd[args[0]]; ok { + if app.conn == nil && args[0] != "help" && args[0] != "connect" { + app.print("Need to make a connection first using \"connect\"") + continue + } + f(args[1:]) + } else { + fmt.Printf("Command %s does not exist\n", args[0]) + } + } +} + +func (app *App) help(args []string) { + for cmd := range app.cmd { + app.print(cmd) + } +} + +func (app *App) connect(args []string) { + if len(args) != 1 { + app.print("Must supply address to connect to") + return + } + + conn, err := ftp.Dial(args[0]) + if err != nil { + app.print(err) + return + } + + app.conn = conn + + if app.conn.IsHerculesSupported() { + app.print("This server supports Hercules up- and downloads, mode H for faster file transfers.") + } +} + +func (app *App) login(args []string) { + if len(args) != 2 { + app.print("Must supply username and password") + return + } + + err := app.conn.Login(args[0], args[1]) + if err != nil { + app.print(err) + } +} + +func (app *App) ls(args []string) { + path := "" + if len(args) == 1 { + path = args[0] + } + + entries, err := app.conn.List(path) + + if err != nil { + app.print(err) + return + } + + for _, entry := range entries { + app.print(entry.Name) + } +} + +func (app *App) cd(args []string) { + if len(args) != 1 { + app.print("Must supply one argument for directory change") + return + } + + err := app.conn.ChangeDir(args[0]) + if err != nil { + app.print(err) + } +} + +func (app *App) mkdir(args []string) { + if len(args) != 1 { + app.print("Must supply one argument for directory name") + return + } + + err := app.conn.MakeDir(args[0]) + if err != nil { + app.print(err) + } +} + +func (app *App) pwd(args []string) { + cur, err := app.conn.CurrentDir() + if err != nil { + app.print(err) + } + app.print(cur) +} + +func (app *App) mode(args []string) { + if len(args) != 1 { + app.print("Must supply one argument for mode, [S]tream, [E]xtended or [H]ercules") + return + } + + err := app.conn.Mode([]byte(args[0])[0]) + if err != nil { + app.print(err) + } +} + +func (app *App) retr(args []string) { + if len(args) < 2 || len(args) > 4 { + app.print("Must supply one argument for source and one for destination, optionally one for offset and one for length") + return + } + + remotePath := args[0] + localPath := args[1] + offset := -1 + length := -1 + + var resp *ftp.Response + var err error + + if len(args) >= 3 { + offset, err = strconv.Atoi(args[2]) + if err != nil { + app.print("Failed to parse offset") + return + } + } + + if len(args) == 4 { + length, err = strconv.Atoi(args[3]) + if err != nil { + app.print("Failed to parse length") + return + } + } + + if app.conn.IsModeHercules() { // With Hercules, separation of data transmission and persistence is not possible + if offset != -1 && length != -1 { + app.print("ERET not supported with Hercules") + } else if offset != -1 { + err = app.conn.RetrHerculesFrom(app.herculesBinary, remotePath, localPath, int64(offset)) + } else { + err = app.conn.RetrHercules(app.herculesBinary, remotePath, localPath) + } + if err != nil { + app.print(err) + } + } else { + if offset != -1 && length != -1 { + resp, err = app.conn.Eret(remotePath, offset, length) + } else if offset != -1 { + resp, err = app.conn.RetrFrom(remotePath, uint64(offset)) + } else { + resp, err = app.conn.Retr(remotePath) + } + + if err != nil { + app.print(err) + return + } + defer resp.Close() + + f, err := os.Create(localPath) + if err != nil { + app.print(err) + return + } + defer f.Close() + + n, err := io.Copy(f, resp) + if err != nil { + app.print(err) + } else { + app.print(fmt.Sprintf("Received %d bytes", n)) + } + } +} + +func (app *App) stor(args []string) { + if len(args) != 2 { + app.print("Must supply one argument for source and one for destination") + return + } + + var err error + if app.conn.IsModeHercules() { // With Hercules, separation of data transmission and persistence is not possible + err = app.conn.StorHercules(app.herculesBinary, args[0], args[1]) + } else { + var f *os.File + f, err = os.Open(args[0]) + if err != nil { + app.print(err) + return + } + + err = app.conn.Stor(args[1], f) + } + if err != nil { + app.print(err) + } +} + +func (app *App) quit(args []string) { + err := app.conn.Quit() + if err != nil { + app.print(err) + } else { + app.print("Goodbye") + } + + os.Exit(0) +} diff --git a/ftpd/README.md b/ftpd/README.md new file mode 100644 index 00000000..c2c605f4 --- /dev/null +++ b/ftpd/README.md @@ -0,0 +1,20 @@ +# scion-ftpd + +This is a sample FTP server for testing and demonstrating usage of FTP on the SCION network. Build this application +from [scion-apps](../../) using the command `make scion-ftpd` + +``` +$ scionftp_server -h +Usage of scion-ftpd: + -hercules string + Enable RETR_HERCULES using the Hercules binary specified + In Hercules mode, scionFTP checks the following directories for Hercules config files: ., /etc, /etc/scion-ftp + -pass string + Password for login (omit for anonymous FTP) + -port uint + Port (default 2121) + -root string + Root directory to serve + -user string + Username for login (omit for anonymous FTP) +``` diff --git a/ftpd/internal/core/LICENSE b/ftpd/internal/core/LICENSE new file mode 100644 index 00000000..091e8c01 --- /dev/null +++ b/ftpd/internal/core/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2018 Goftp Authors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/ftpd/internal/core/README.md b/ftpd/internal/core/README.md new file mode 100644 index 00000000..20280543 --- /dev/null +++ b/ftpd/internal/core/README.md @@ -0,0 +1,66 @@ +# scionftp server library + +Server package for FTP + GridFTP extension, adapted to the SCION network. Forked +from [elwin/scionFTP](https://github.com/elwin/scionFTP). + +## Usage + +To boot a FTP server you will need to provide a driver that speaks to +your persistence layer - the required driver contract is in [the +documentation](https://godoc.org/github.com/elwin/ScionFTP/server#Driver). + +Look at the [file driver](../file-driver) to see +an example of how to build a backend. + +There is a [sample ftp server](scionftp_server) as a demo. You can build it with this +command: + + go install github.com/elwin/scionFTP/server/scionftp_server + +Then run it if you have add $GOPATH to your $PATH: + + scionftp_server -root /tmp + +And finally, connect to the server with any FTP client and the following +details: + + host: 127.0.0.1 + port: 2121 + username: admin + password: 123456 + +This uses the file driver mentioned above to serve files. + + +## Contributing + +All suggestions and patches welcome, preferably via a git repository I can pull from. +If this library proves useful to you, please let me know. + +## Further Reading + +There are a range of RFCs that together specify the FTP protocol. In chronological +order, the more useful ones are: + +* [http://tools.ietf.org/rfc/rfc959.txt](http://tools.ietf.org/rfc/rfc959.txt) +* [http://tools.ietf.org/rfc/rfc1123.txt](http://tools.ietf.org/rfc/rfc1123.txt) +* [http://tools.ietf.org/rfc/rfc2228.txt](http://tools.ietf.org/rfc/rfc2228.txt) +* [http://tools.ietf.org/rfc/rfc2389.txt](http://tools.ietf.org/rfc/rfc2389.txt) +* [http://tools.ietf.org/rfc/rfc2428.txt](http://tools.ietf.org/rfc/rfc2428.txt) +* [http://tools.ietf.org/rfc/rfc3659.txt](http://tools.ietf.org/rfc/rfc3659.txt) +* [http://tools.ietf.org/rfc/rfc4217.txt](http://tools.ietf.org/rfc/rfc4217.txt) + +For an english summary that's somewhat more legible than the RFCs, and provides +some commentary on what features are actually useful or relevant 24 years after +RFC959 was published: + +* [http://cr.yp.to/ftp.html](http://cr.yp.to/ftp.html) + +For a history lesson, check out Appendix III of RCF959. It lists the preceding +(obsolete) RFC documents that relate to file transfers, including the ye old +RFC114 from 1971, "A File Transfer Protocol" + +This library is heavily based on [em-ftpd](https://github.com/yob/em-ftpd), an FTPd +framework with similar design goals within the ruby and EventMachine ecosystems. It +worked well enough, but you know, callbacks and event loops make me something +something. diff --git a/ftpd/internal/core/auth.go b/ftpd/internal/core/auth.go new file mode 100644 index 00000000..064ef87a --- /dev/null +++ b/ftpd/internal/core/auth.go @@ -0,0 +1,53 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// +// Copyright 2021 ETH Zurich modifications to add anonymous authentication + +package core + +import ( + "crypto/subtle" +) + +// Auth is an interface to auth your ftp user login. +type Auth interface { + // Verifies login credentials + CheckPasswd(string, string) (bool, error) + // Checks if the supplied user is authorized to perform actions that require authentication + IsAuthorized(string) bool +} + +var ( + _ Auth = &SimpleAuth{} +) + +// SimpleAuth implements Auth interface to provide a memory user login auth +type SimpleAuth struct { + Name string + Password string +} + +// CheckPasswd will check user's password +func (a *SimpleAuth) CheckPasswd(name, pass string) (bool, error) { + return constantTimeEquals(name, a.Name) && constantTimeEquals(pass, a.Password), nil +} + +func constantTimeEquals(a, b string) bool { + return len(a) == len(b) && subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1 +} + +func (a *SimpleAuth) IsAuthorized(s string) bool { + return s != "" +} + +// AnonymousAuth implements Auth interface to enable authentication commands when using anonymous FTP +type AnonymousAuth struct{} + +func (a AnonymousAuth) CheckPasswd(name, pass string) (bool, error) { + return name == "anonymous", nil +} + +func (a AnonymousAuth) IsAuthorized(name string) bool { + return true +} diff --git a/ftpd/internal/core/cmd.go b/ftpd/internal/core/cmd.go new file mode 100644 index 00000000..be951e95 --- /dev/null +++ b/ftpd/internal/core/cmd.go @@ -0,0 +1,1435 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package core + +import ( + "fmt" + "log" + "net" + "os" + "strconv" + "strings" + "syscall" + "time" + + "github.com/scionproto/scion/go/lib/snet" + + "github.com/netsec-ethz/scion-apps/internal/ftp/hercules" + "github.com/netsec-ethz/scion-apps/internal/ftp/mode" + "github.com/netsec-ethz/scion-apps/internal/ftp/socket" + "github.com/netsec-ethz/scion-apps/internal/ftp/striping" + "github.com/netsec-ethz/scion-apps/pkg/appnet" +) + +type Command interface { + IsExtend() bool + RequireParam() bool + RequireAuth() bool + Execute(*Conn, string) +} + +type commandMap map[string]Command + +var ( + commands = commandMap{ + "ADAT": commandAdat{}, + "ALLO": commandAllo{}, + "APPE": commandAppe{}, + "CDUP": commandCdup{}, + "CWD": commandCwd{}, + "CCC": commandCcc{}, + "CONF": commandConf{}, + "DELE": commandDele{}, + "ENC": commandEnc{}, + "EPRT": commandEprt{}, + "EPSV": commandEpsv{}, + "FEAT": commandFeat{}, + "LIST": commandList{}, + "LPRT": commandLprt{}, + "NLST": commandNlst{}, + "MDTM": commandMdtm{}, + "MIC": commandMic{}, + "MKD": commandMkd{}, + "MODE": commandMode{}, + "NOOP": commandNoop{}, + "OPTS": commandOpts{}, + "PASS": commandPass{}, + "PASV": commandPasv{}, + "PBSZ": commandPbsz{}, + "PORT": commandPort{}, + "PROT": commandProt{}, + "PWD": commandPwd{}, + "QUIT": commandQuit{}, + "RETR": commandRetr{}, + "REST": commandRest{}, + "RNFR": commandRnfr{}, + "RNTO": commandRnto{}, + "RMD": commandRmd{}, + "SIZE": commandSize{}, + "STOR": commandStor{}, + "STRU": commandStru{}, + "SYST": commandSyst{}, + "TYPE": commandType{}, + "USER": commandUser{}, + "XCUP": commandCdup{}, + "XCWD": commandCwd{}, + "XMKD": commandMkd{}, + "XPWD": commandPwd{}, + "XRMD": commandRmd{}, + "SPAS": commandSpas{}, + "ERET": commandEret{}, + } +) + +// commandAllo responds to the ALLO FTP command. +// +// This is essentially a ping from the scionftp so we just respond with an +// basic OK message. +type commandAllo struct{} + +func (cmd commandAllo) IsExtend() bool { + return false +} + +func (cmd commandAllo) RequireParam() bool { + return false +} + +func (cmd commandAllo) RequireAuth() bool { + return false +} + +func (cmd commandAllo) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(202, "Obsolete") +} + +// commandAppe responds to the APPE FTP command. It allows the user to upload a +// new file but always append if file exists otherwise create one. +type commandAppe struct{} + +func (cmd commandAppe) IsExtend() bool { + return false +} + +func (cmd commandAppe) RequireParam() bool { + return true +} + +func (cmd commandAppe) RequireAuth() bool { + return true +} + +func (cmd commandAppe) Execute(conn *Conn, param string) { + targetPath := conn.buildPath(param) + _, _ = conn.writeMessage(150, "Data transfer starting") + + bytes, err := conn.driver.PutFile(targetPath, conn.dataConn, true) + if err == nil { + msg := "OK, received " + strconv.Itoa(int(bytes)) + " bytes" + _, _ = conn.writeMessage(226, msg) + } else { + _, _ = conn.writeMessage(450, fmt.Sprint("error during transfer: ", err)) + } +} + +type commandOpts struct{} + +func (cmd commandOpts) IsExtend() bool { + return false +} + +func (cmd commandOpts) RequireParam() bool { + return false +} + +func (cmd commandOpts) RequireAuth() bool { + return false +} +func (cmd commandOpts) Execute(conn *Conn, param string) { + parts := strings.Fields(param) + if len(parts) != 2 { + _, _ = conn.writeMessage(550, "Unknown params") + return + } + + switch strings.ToUpper(parts[0]) { + case "UTF8": + if strings.ToUpper(parts[1]) == "ON" { + _, _ = conn.writeMessage(200, "UTF8 mode enabled") + } else { + _, _ = conn.writeMessage(550, "Unsupported non-utf8 mode") + } + case "RETR": + parallelism, blockSize, err := ParseOptions(parts[1]) + if err != nil { + _, _ = conn.writeMessage(550, fmt.Sprintf("failed to parse options: %s", err)) + } else { + conn.parallelism = parallelism + conn.blockSize = blockSize + _, _ = conn.writeMessage(200, fmt.Sprintf("Parallelism set to %d", parallelism)) + } + default: + _, _ = conn.writeMessage(550, "Unknown params") + } + +} + +func ParseOptions(param string) (parallelism, blockSize int, err error) { + parts := strings.Split(strings.TrimRight(param, ";"), ";") + for _, part := range parts { + splitted := strings.Split(part, "=") + if len(splitted) != 2 { + err = fmt.Errorf("unknown params") + return + } + + switch strings.ToUpper(splitted[0]) { + case "PARALLELISM": + parallelism, err = strconv.Atoi(splitted[1]) + if err != nil || parallelism < 1 { + err = fmt.Errorf("unknown params") + return + } + case "STRIPELAYOUT": + if strings.ToUpper(splitted[1]) != "BLOCKED" { + err = fmt.Errorf("only blocked mode supported") + return + } + case "BLOCKSIZE": + blockSize, err = strconv.Atoi(splitted[1]) + if err != nil || blockSize < 1 { + err = fmt.Errorf("unknown params") + return + } + } + } + + return +} + +type commandFeat struct{} + +func (cmd commandFeat) IsExtend() bool { + return false +} + +func (cmd commandFeat) RequireParam() bool { + return false +} + +func (cmd commandFeat) RequireAuth() bool { + return false +} + +var ( + feats = "Extensions supported:\n%s" + featCmds = " UTF8\n" +) + +func init() { + for k, v := range commands { + if v.IsExtend() { + featCmds = featCmds + " " + k + "\n" + } + } +} + +func (cmd commandFeat) Execute(conn *Conn, param string) { + _, _ = conn.writeMessageMultiline(211, conn.server.feats) +} + +// cmdCdup responds to the CDUP FTP command. +// +// Allows the scionftp change their current directory to the parent. +type commandCdup struct{} + +func (cmd commandCdup) IsExtend() bool { + return false +} + +func (cmd commandCdup) RequireParam() bool { + return false +} + +func (cmd commandCdup) RequireAuth() bool { + return true +} + +func (cmd commandCdup) Execute(conn *Conn, param string) { + otherCmd := &commandCwd{} + otherCmd.Execute(conn, "..") +} + +// commandCwd responds to the CWD FTP command. It allows the scionftp to change the +// current working directory. +type commandCwd struct{} + +func (cmd commandCwd) IsExtend() bool { + return false +} + +func (cmd commandCwd) RequireParam() bool { + return true +} + +func (cmd commandCwd) RequireAuth() bool { + return true +} + +func (cmd commandCwd) Execute(conn *Conn, param string) { + path := conn.buildPath(param) + err := conn.driver.ChangeDir(path) + if err == nil { + conn.namePrefix = path + _, _ = conn.writeMessage(250, "Directory changed to "+path) + } else { + _, _ = conn.writeMessage(550, fmt.Sprint("Directory change to ", path, " failed: ", err)) + } +} + +// commandDele responds to the DELE FTP command. It allows the scionftp to delete +// a file +type commandDele struct{} + +func (cmd commandDele) IsExtend() bool { + return false +} + +func (cmd commandDele) RequireParam() bool { + return true +} + +func (cmd commandDele) RequireAuth() bool { + return true +} + +func (cmd commandDele) Execute(conn *Conn, param string) { + path := conn.buildPath(param) + err := conn.driver.DeleteFile(path) + if err == nil { + _, _ = conn.writeMessage(250, "File deleted") + } else { + _, _ = conn.writeMessage(550, fmt.Sprint("File delete failed: ", err)) + } +} + +// commandEprt responds to the EPRT FTP command. It allows the scionftp to +// request an active data socket with more options than the original PORT +// command. It mainly adds ipv6 support. +type commandEprt struct{} + +func (cmd commandEprt) IsExtend() bool { + return true +} + +func (cmd commandEprt) RequireParam() bool { + return true +} + +func (cmd commandEprt) RequireAuth() bool { + return true +} + +func (cmd commandEprt) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(502, "Active mode not supported, use passive mode instead") +} + +// commandLprt responds to the LPRT FTP command. It allows the scionftp to +// request an active data socket with more options than the original PORT +// command. FTP Operation Over Big Address Records. +type commandLprt struct{} + +func (cmd commandLprt) IsExtend() bool { + return true +} + +func (cmd commandLprt) RequireParam() bool { + return true +} + +func (cmd commandLprt) RequireAuth() bool { + return true +} + +func (cmd commandLprt) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(502, "Active mode not supported, use passive mode instead") + +} + +// commandEpsv responds to the EPSV FTP command. It allows the speedtest_client to +// request a passive data socket with more options than the original PASV +// command. It mainly adds ipv6 support, although we don't support that yet. +type commandEpsv struct{} + +func (cmd commandEpsv) IsExtend() bool { + return true +} + +func (cmd commandEpsv) RequireParam() bool { + return false +} + +func (cmd commandEpsv) RequireAuth() bool { + return true +} + +func (cmd commandEpsv) Execute(conn *Conn, param string) { + + listener, err := conn.NewListener() + + if err != nil { + log.Println(err) + _, _ = conn.writeMessage(425, "Data connection failed") + return + } + msg := fmt.Sprintf("Entering Extended Passive Mode (|||%d|)", listener.QuicListener.Addr().(*net.UDPAddr).Port) + _, _ = conn.writeMessage(229, msg) + + sock, err := listener.Accept() + if err != nil { + _, _ = conn.writeMessage(426, "Connection closed, failed to open data connection") + return + } + conn.dataConn = socket.DelayedCloserSocket{Conn: sock, Closer: listener, Duration: 120 * time.Second} + +} + +// commandList responds to the LIST FTP command. It allows the speedtest_client to retrieve +// a detailed listing of the contents of a directory. +type commandList struct{} + +func (cmd commandList) IsExtend() bool { + return false +} + +func (cmd commandList) RequireParam() bool { + return false +} + +func (cmd commandList) RequireAuth() bool { + return true +} + +func (cmd commandList) Execute(conn *Conn, param string) { + path := conn.buildPath(parseListParam(param)) + info, err := conn.driver.Stat(path) + if err != nil { + _, _ = conn.writeMessage(550, err.Error()) + return + } + + if info == nil { + conn.logger.Printf(conn.sessionID, "%s: no such file or directory.\n", path) + return + } + var files []FileInfo + if info.IsDir() { + err = conn.driver.ListDir(path, func(f FileInfo) error { + files = append(files, f) + return nil + }) + if err != nil { + _, _ = conn.writeMessage(550, err.Error()) + return + } + } else { + files = append(files, info) + } + + _, _ = conn.writeMessage(150, "Opening ASCII mode data connection for file list") + conn.sendOutofbandData(listFormatter(files).Detailed()) +} + +func parseListParam(param string) (path string) { + if len(param) == 0 { + path = param + } else { + fields := strings.Fields(param) + i := 0 + for _, field := range fields { + if !strings.HasPrefix(field, "-") { + break + } + i = strings.LastIndex(param, " "+field) + len(field) + 1 + } + path = strings.TrimLeft(param[i:], " ") //Get all the path even with space inside + } + return path +} + +// commandNlst responds to the NLST FTP command. It allows the speedtest_client to +// retrieve a list of filenames in the current directory. +type commandNlst struct{} + +func (cmd commandNlst) IsExtend() bool { + return false +} + +func (cmd commandNlst) RequireParam() bool { + return false +} + +func (cmd commandNlst) RequireAuth() bool { + return true +} + +func (cmd commandNlst) Execute(conn *Conn, param string) { + path := conn.buildPath(parseListParam(param)) + info, err := conn.driver.Stat(path) + if err != nil { + _, _ = conn.writeMessage(550, err.Error()) + return + } + if !info.IsDir() { + _, _ = conn.writeMessage(550, param+" is not a directory") + return + } + + var files []FileInfo + err = conn.driver.ListDir(path, func(f FileInfo) error { + files = append(files, f) + return nil + }) + if err != nil { + _, _ = conn.writeMessage(550, err.Error()) + return + } + _, _ = conn.writeMessage(150, "Opening ASCII mode data connection for file list") + conn.sendOutofbandData(listFormatter(files).Short()) +} + +// commandMdtm responds to the MDTM FTP command. It allows the speedtest_client to +// retrieve the last modified time of a file. +type commandMdtm struct{} + +func (cmd commandMdtm) IsExtend() bool { + return false +} + +func (cmd commandMdtm) RequireParam() bool { + return true +} + +func (cmd commandMdtm) RequireAuth() bool { + return true +} + +func (cmd commandMdtm) Execute(conn *Conn, param string) { + path := conn.buildPath(param) + stat, err := conn.driver.Stat(path) + if err == nil { + _, _ = conn.writeMessage(213, stat.ModTime().Format("20060102150405")) + } else { + _, _ = conn.writeMessage(450, "File not available") + } +} + +// commandMkd responds to the MKD FTP command. It allows the speedtest_client to create +// a new directory +type commandMkd struct{} + +func (cmd commandMkd) IsExtend() bool { + return false +} + +func (cmd commandMkd) RequireParam() bool { + return true +} + +func (cmd commandMkd) RequireAuth() bool { + return true +} + +func (cmd commandMkd) Execute(conn *Conn, param string) { + path := conn.buildPath(param) + err := conn.driver.MakeDir(path) + if err == nil { + _, _ = conn.writeMessage(257, "Directory created") + } else { + _, _ = conn.writeMessage(550, fmt.Sprint("Action not taken: ", err)) + } +} + +// cmdMode responds to the MODE FTP command. +// +// the original FTP spec had various options for hosts to negotiate how data +// would be sent over the data socket, In reality these days (S)tream mode +// is all that is used for the mode - data is just streamed down the data +// socket unchanged. +type commandMode struct{} + +func (cmd commandMode) IsExtend() bool { + return false +} + +func (cmd commandMode) RequireParam() bool { + return true +} + +func (cmd commandMode) RequireAuth() bool { + return true +} + +func (cmd commandMode) Execute(conn *Conn, param string) { + if strings.ToUpper(param) == "S" { + conn.mode = mode.Stream + + } else if strings.ToUpper(param) == "E" { + conn.mode = mode.ExtendedBlockMode + conn.parallelism = 4 + conn.blockSize = 500 + + } else if strings.ToUpper(param) == "H" { + if conn.server.HerculesBinary == "" { + conn.writeOrLog(504, "Hercules mode not supported") + return + } + conn.mode = mode.Hercules + + } else { + _, _ = conn.writeMessage(504, "MODE is an obsolete command, only (S)tream (E)xtended and (H)ercules Mode supported") + return + } + _, _ = conn.writeMessage(200, "OK") +} + +// cmdNoop responds to the NOOP FTP command. +// +// This is essentially a ping from the speedtest_client so we just respond with an +// basic 200 message. +type commandNoop struct{} + +func (cmd commandNoop) IsExtend() bool { + return false +} + +func (cmd commandNoop) RequireParam() bool { + return false +} + +func (cmd commandNoop) RequireAuth() bool { + return false +} + +func (cmd commandNoop) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(200, "OK") +} + +// commandPass respond to the PASS FTP command by asking the driver if the +// supplied username and password are valid +type commandPass struct{} + +func (cmd commandPass) IsExtend() bool { + return false +} + +func (cmd commandPass) RequireParam() bool { + return true +} + +func (cmd commandPass) RequireAuth() bool { + return false +} + +func (cmd commandPass) Execute(conn *Conn, param string) { + ok, err := conn.server.Auth.CheckPasswd(conn.reqUser, param) + if err != nil { + _, _ = conn.writeMessage(550, "Checking password error") + return + } + + if ok { + conn.user = conn.reqUser + conn.reqUser = "" + _, _ = conn.writeMessage(230, "Password ok, continue") + } else { + _, _ = conn.writeMessage(530, "Incorrect password, not logged in") + } +} + +// commandPasv responds to the PASV FTP command. +// +// The speedtest_client is requesting us to open a new TCP listing socket and wait for them +// to connect to it. +type commandPasv struct{} + +func (cmd commandPasv) IsExtend() bool { + return false +} + +func (cmd commandPasv) RequireParam() bool { + return false +} + +func (cmd commandPasv) RequireAuth() bool { + return true +} + +func (cmd commandPasv) Execute(conn *Conn, param string) { + commandEpsv{}.Execute(conn, param) +} + +// commandPort responds to the PORT FTP command. +// +// The speedtest_client has opened a listening socket for sending out of band data and +// is requesting that we connect to it +type commandPort struct{} + +func (cmd commandPort) IsExtend() bool { + return false +} + +func (cmd commandPort) RequireParam() bool { + return true +} + +func (cmd commandPort) RequireAuth() bool { + return true +} + +func (cmd commandPort) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(502, "Active mode not supported, use passive mode instead") +} + +// commandPwd responds to the PWD FTP command. +// +// Tells the speedtest_client what the current working directory is. +type commandPwd struct{} + +func (cmd commandPwd) IsExtend() bool { + return false +} + +func (cmd commandPwd) RequireParam() bool { + return false +} + +func (cmd commandPwd) RequireAuth() bool { + return true +} + +func (cmd commandPwd) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(257, "\""+conn.namePrefix+"\" is the current directory") +} + +// CommandQuit responds to the QUIT FTP command. The speedtest_client has requested the +// connection be closed. +type commandQuit struct{} + +func (cmd commandQuit) IsExtend() bool { + return false +} + +func (cmd commandQuit) RequireParam() bool { + return false +} + +func (cmd commandQuit) RequireAuth() bool { + return false +} + +func (cmd commandQuit) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(221, "Goodbye") + conn.Close() +} + +// commandRetr responds to the RETR FTP command. It allows the speedtest_client to +// download a file. +type commandRetr struct{} + +func (cmd commandRetr) IsExtend() bool { + return false +} + +func (cmd commandRetr) RequireParam() bool { + return true +} + +func (cmd commandRetr) RequireAuth() bool { + return true +} + +func (cmd commandRetr) Execute(conn *Conn, param string) { + path := conn.buildPath(param) + defer func() { + conn.lastFilePos = 0 + conn.appendData = false + }() + + if conn.mode == 'H' { + cmd.executeHercules(conn, path) + } else { + cmd.executeTraditional(conn, path) + } +} + +func (cmd commandRetr) executeTraditional(conn *Conn, path string) { + bytes, data, err := conn.driver.GetFile(path, conn.lastFilePos) + if err == nil { + defer func() { _ = data.Close() }() + _, _ = conn.writeMessage(150, fmt.Sprintf("Data transfer starting %v bytes", bytes)) + err = conn.sendOutofBandDataWriter(data) + if err != nil { + _, _ = conn.writeMessage(551, "Error reading file") + } + } else { + _, _ = conn.writeMessage(551, "File not available") + } +} + +func (cmd commandRetr) executeHercules(conn *Conn, path string) { + if conn.dataConn == nil { + log.Print("No data connection open") + conn.writeOrLog(425, "Can't open data connection") + return + } + defer func() { + _ = conn.dataConn.Close() + conn.dataConn = nil + }() + + // check file access as unprivileged user + realPath := conn.driver.RealPath(path) + f, err := os.Open(realPath) + if err != nil { + conn.writeOrLog(551, "File not available for download") + return + } + _ = f.Close() + + if !conn.server.herculesLock.tryLockTimeout(5 * time.Second) { + conn.writeOrLog(425, "All Hercules units busy - please try again later") + return + } + defer conn.server.herculesLock.unlock() + + command, err := hercules.PrepareHerculesSendCommand(conn.server.HerculesBinary, conn.server.HerculesConfig, conn.dataConn.LocalAddr().(*net.UDPAddr), conn.dataConn.RemoteAddr().(*snet.UDPAddr), realPath, conn.lastFilePos) + if err != nil { + log.Printf("could not run hercules: %s", err) + conn.writeOrLog(425, "Can't open data connection") + return + } + command.Stderr = os.Stderr + command.Stdout = os.Stdout + + _, err = conn.writeMessage(150, "File status okay; about to open data connection.") + if err != nil { + log.Printf("could not write response: %s", err.Error()) + return + } + + log.Printf("run Hercules: %s", command) + err = command.Run() + if err != nil { + // TODO improve error handling + log.Printf("could not execute Hercules: %s", err) + conn.writeOrLog(551, "Hercules returned an error") + } else { + conn.writeOrLog(226, "Hercules transfer complete") + } +} + +type commandRest struct{} + +func (cmd commandRest) IsExtend() bool { + return false +} + +func (cmd commandRest) RequireParam() bool { + return true +} + +func (cmd commandRest) RequireAuth() bool { + return true +} + +func (cmd commandRest) Execute(conn *Conn, param string) { + var err error + conn.lastFilePos, err = strconv.ParseInt(param, 10, 64) + if err != nil { + _, _ = conn.writeMessage(551, "File not available") + return + } + + conn.appendData = true + + _, _ = conn.writeMessage(350, fmt.Sprint("Start transfer from ", conn.lastFilePos)) +} + +// commandRnfr responds to the RNFR FTP command. It's the first of two commands +// required for a speedtest_client to rename a file. +type commandRnfr struct{} + +func (cmd commandRnfr) IsExtend() bool { + return false +} + +func (cmd commandRnfr) RequireParam() bool { + return true +} + +func (cmd commandRnfr) RequireAuth() bool { + return true +} + +func (cmd commandRnfr) Execute(conn *Conn, param string) { + conn.renameFrom = conn.buildPath(param) + _, _ = conn.writeMessage(350, "Requested file action pending further information.") +} + +// cmdRnto responds to the RNTO FTP command. It's the second of two commands +// required for a speedtest_client to rename a file. +type commandRnto struct{} + +func (cmd commandRnto) IsExtend() bool { + return false +} + +func (cmd commandRnto) RequireParam() bool { + return true +} + +func (cmd commandRnto) RequireAuth() bool { + return true +} + +func (cmd commandRnto) Execute(conn *Conn, param string) { + toPath := conn.buildPath(param) + err := conn.driver.Rename(conn.renameFrom, toPath) + defer func() { + conn.renameFrom = "" + }() + + if err == nil { + _, _ = conn.writeMessage(250, "File renamed") + } else { + _, _ = conn.writeMessage(550, fmt.Sprint("Action not taken: ", err)) + } +} + +// cmdRmd responds to the RMD FTP command. It allows the speedtest_client to delete a +// directory. +type commandRmd struct{} + +func (cmd commandRmd) IsExtend() bool { + return false +} + +func (cmd commandRmd) RequireParam() bool { + return true +} + +func (cmd commandRmd) RequireAuth() bool { + return true +} + +func (cmd commandRmd) Execute(conn *Conn, param string) { + path := conn.buildPath(param) + err := conn.driver.DeleteDir(path) + if err == nil { + _, _ = conn.writeMessage(250, "Directory deleted") + } else { + _, _ = conn.writeMessage(550, fmt.Sprint("Directory delete failed: ", err)) + } +} + +type commandAdat struct{} + +func (cmd commandAdat) IsExtend() bool { + return false +} + +func (cmd commandAdat) RequireParam() bool { + return true +} + +func (cmd commandAdat) RequireAuth() bool { + return true +} + +func (cmd commandAdat) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(550, "Action not taken") +} + +type commandCcc struct{} + +func (cmd commandCcc) IsExtend() bool { + return false +} + +func (cmd commandCcc) RequireParam() bool { + return true +} + +func (cmd commandCcc) RequireAuth() bool { + return true +} + +func (cmd commandCcc) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(550, "Action not taken") +} + +type commandEnc struct{} + +func (cmd commandEnc) IsExtend() bool { + return false +} + +func (cmd commandEnc) RequireParam() bool { + return true +} + +func (cmd commandEnc) RequireAuth() bool { + return true +} + +func (cmd commandEnc) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(550, "Action not taken") +} + +type commandMic struct{} + +func (cmd commandMic) IsExtend() bool { + return false +} + +func (cmd commandMic) RequireParam() bool { + return true +} + +func (cmd commandMic) RequireAuth() bool { + return true +} + +func (cmd commandMic) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(550, "Action not taken") +} + +type commandPbsz struct{} + +func (cmd commandPbsz) IsExtend() bool { + return false +} + +func (cmd commandPbsz) RequireParam() bool { + return true +} + +func (cmd commandPbsz) RequireAuth() bool { + return false +} + +func (cmd commandPbsz) Execute(conn *Conn, param string) { + if param == "0" { + _, _ = conn.writeMessage(200, "OK") + } else { + _, _ = conn.writeMessage(550, "Action not taken") + } +} + +type commandProt struct{} + +func (cmd commandProt) IsExtend() bool { + return false +} + +func (cmd commandProt) RequireParam() bool { + return true +} + +func (cmd commandProt) RequireAuth() bool { + return false +} + +func (cmd commandProt) Execute(conn *Conn, param string) { + if param == "P" { + _, _ = conn.writeMessage(200, "OK") + } else { + _, _ = conn.writeMessage(550, "Action not taken") + } +} + +type commandConf struct{} + +func (cmd commandConf) IsExtend() bool { + return false +} + +func (cmd commandConf) RequireParam() bool { + return true +} + +func (cmd commandConf) RequireAuth() bool { + return true +} + +func (cmd commandConf) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(550, "Action not taken") +} + +// commandSize responds to the SIZE FTP command. It returns the size of the +// requested path in bytes. +type commandSize struct{} + +func (cmd commandSize) IsExtend() bool { + return false +} + +func (cmd commandSize) RequireParam() bool { + return true +} + +func (cmd commandSize) RequireAuth() bool { + return true +} + +func (cmd commandSize) Execute(conn *Conn, param string) { + path := conn.buildPath(param) + stat, err := conn.driver.Stat(path) + if err != nil { + log.Printf("Size: error(%s)", err) + _, _ = conn.writeMessage(450, fmt.Sprint("path", path, "not found")) + } else { + _, _ = conn.writeMessage(213, strconv.Itoa(int(stat.Size()))) + } +} + +// commandStor responds to the STOR FTP command. It allows the user to upload a +// new file. +type commandStor struct{} + +func (cmd commandStor) IsExtend() bool { + return false +} + +func (cmd commandStor) RequireParam() bool { + return true +} + +func (cmd commandStor) RequireAuth() bool { + return true +} + +func (cmd commandStor) Execute(conn *Conn, param string) { + if conn.lastFilePos != 0 { // not properly implemented as of now (lastFilePos is not used as offset) + _, _ = conn.writeMessage(503, "uploads starting at an offset are not implemented") + return + } + targetPath := conn.buildPath(param) + + if conn.mode == 'H' { + cmd.executeHercules(conn, targetPath) + } else { + cmd.executeTraditional(conn, targetPath) + } +} + +func (cmd commandStor) executeTraditional(conn *Conn, targetPath string) { + _, _ = conn.writeMessage(150, "Data transfer starting") + + defer func() { + conn.appendData = false + }() + + bytes, err := conn.driver.PutFile(targetPath, conn.dataConn, conn.appendData) + if err == nil { + msg := "OK, received " + strconv.Itoa(int(bytes)) + " bytes" + _, _ = conn.writeMessage(226, msg) + } else { + _, _ = conn.writeMessage(450, fmt.Sprint("error during transfer: ", err)) + } +} + +func (cmd commandStor) executeHercules(conn *Conn, path string) { + if conn.dataConn == nil { + log.Print("No data connection open") + conn.writeOrLog(425, "Can't open data connection") + return + } + defer func() { + _ = conn.dataConn.Close() + conn.dataConn = nil + }() + + // check file access as unprivileged user + realPath := conn.driver.RealPath(path) + fileCreated, err := hercules.AssertFileWriteable(realPath) + if err != nil { + conn.writeOrLog(551, "File not available for writing") + return + } + defer func() { + if err != nil && fileCreated { + err = syscall.Unlink(realPath) + if err != nil { + log.Printf("could not delete file: %s", err) + } + } + }() + + if !conn.server.herculesLock.tryLockTimeout(5 * time.Second) { + conn.writeOrLog(425, "All Hercules units busy - please try again later") + return + } + defer conn.server.herculesLock.unlock() + + command, err := hercules.PrepareHerculesRecvCommand(conn.server.HerculesBinary, conn.server.HerculesConfig, conn.dataConn.LocalAddr().(*net.UDPAddr), realPath, -1) + if err != nil { + log.Printf("could not start hercules: %s", err) + conn.writeOrLog(425, "Can't open data connection") + return + } + command.Stderr = os.Stderr + command.Stdout = os.Stdout + + _, err = conn.writeMessage(125, "Data connection already open; transfer starting.") + if err != nil { + log.Printf("could not write response: %s", err.Error()) + return + } + + log.Printf("run Hercules: %s", command) + err = command.Run() + if err != nil { + // TODO improve error handling + log.Printf("could not execute Hercules: %s", err) + conn.writeOrLog(551, "Hercules returned an error") + } else { + conn.writeOrLog(226, "Hercules transfer complete") + fileCreated = false + err = hercules.OwnFile(realPath) + if err != nil { + log.Printf("could not own file: %s", err) + } + } +} + +// commandStru responds to the STRU FTP command. +// +// like the MODE and TYPE commands, stru[cture] dates back to a time when the +// FTP protocol was more aware of the content of the files it was transferring, +// and would sometimes be expected to translate things like EOL markers on the +// fly. +// +// These days files are sent unmodified, and F(ile) mode is the only one we +// really need to support. +type commandStru struct{} + +func (cmd commandStru) IsExtend() bool { + return false +} + +func (cmd commandStru) RequireParam() bool { + return true +} + +func (cmd commandStru) RequireAuth() bool { + return true +} + +func (cmd commandStru) Execute(conn *Conn, param string) { + if strings.ToUpper(param) == "F" { + _, _ = conn.writeMessage(200, "OK") + } else { + _, _ = conn.writeMessage(504, "STRU is an obsolete command") + } +} + +// commandSyst responds to the SYST FTP command by providing a canned response. +type commandSyst struct{} + +func (cmd commandSyst) IsExtend() bool { + return false +} + +func (cmd commandSyst) RequireParam() bool { + return false +} + +func (cmd commandSyst) RequireAuth() bool { + return true +} + +func (cmd commandSyst) Execute(conn *Conn, param string) { + _, _ = conn.writeMessage(215, "UNIX Type: L8") +} + +// commandType responds to the TYPE FTP command. +// +// like the MODE and STRU commands, TYPE dates back to a time when the FTP +// protocol was more aware of the content of the files it was transferring, and +// would sometimes be expected to translate things like EOL markers on the fly. +// +// Valid options were A(SCII), I(mage), E(BCDIC) or LN (for local type). Since +// we plan to just accept bytes from the speedtest_client unchanged, I think Image mode is +// adequate. The RFC requires we accept ASCII mode however, so accept it, but +// ignore it. +type commandType struct{} + +func (cmd commandType) IsExtend() bool { + return false +} + +func (cmd commandType) RequireParam() bool { + return false +} + +func (cmd commandType) RequireAuth() bool { + return true +} + +func (cmd commandType) Execute(conn *Conn, param string) { + if strings.ToUpper(param) == "A" { + _, _ = conn.writeMessage(200, "Type set to ASCII") + } else if strings.ToUpper(param) == "I" { + _, _ = conn.writeMessage(200, "Type set to binary") + } else { + _, _ = conn.writeMessage(500, "Invalid type") + } +} + +// commandUser responds to the USER FTP command by asking for the password +type commandUser struct{} + +func (cmd commandUser) IsExtend() bool { + return false +} + +func (cmd commandUser) RequireParam() bool { + return true +} + +func (cmd commandUser) RequireAuth() bool { + return false +} + +func (cmd commandUser) Execute(conn *Conn, param string) { + conn.reqUser = param + _, _ = conn.writeMessage(331, "User name ok, password required") +} + +// GridFTP Extensions (https://www.ogf.org/documents/GFD.20.pdf) + +// Striped Passive +// +// This command is analogous to the PASV command, but allows an array of +// host/port connections to be returned. This enables STRIPING, that is, +// multiple network endpoints (multi-homed hosts, or multiple hosts) to +// participate in the transfer. +type commandSpas struct{} + +func (cmd commandSpas) IsExtend() bool { + return true +} + +func (cmd commandSpas) RequireParam() bool { + return false +} + +func (cmd commandSpas) RequireAuth() bool { + return true +} + +func (cmd commandSpas) Execute(conn *Conn, param string) { + sockets := make([]net.Conn, conn.parallelism) + var err error + + line := "Entering Striped Passive Mode\n" + + listener, err := conn.NewListener() + if err != nil { + _, _ = conn.writeMessage(425, "Data connection failed") + return + } + + listenAddr := appnet.DefNetwork().IA.String() + "," + listener.QuicListener.Addr().(*net.UDPAddr).String() + for i := 0; i < conn.parallelism; i++ { + line += " " + listenAddr + "\r\n" + } + + _, _ = conn.writeMessageMultiline(229, line) + + for i := range sockets { + connection, err := listener.Accept() + if err != nil { + _, _ = conn.writeMessage(426, "Connection closed, failed to open data connection") + + // Close already opened sockets + for j := 0; j < i; j++ { + _ = sockets[i].Close() + } + return + } + sockets[i] = connection + } + + conn.dataConn = socket.DelayedCloserSocket{ + Conn: striping.NewMultiSocket(sockets, conn.blockSize), + Closer: listener, + Duration: 120 * time.Second, + } +} + +type commandEret struct{} + +func (commandEret) IsExtend() bool { + return true +} + +func (commandEret) RequireParam() bool { + return true +} + +func (commandEret) RequireAuth() bool { + return true +} + +// TODO: Handle conn.lastFilePos yet +func (commandEret) Execute(conn *Conn, param string) { + + params := strings.Split(param, " ") + module := strings.Split(params[0], "=") + moduleName := module[0] + moduleParams := strings.Split(strings.Trim(module[1], "\""), ",") + offset, err := strconv.Atoi(moduleParams[0]) + if err != nil { + _, _ = conn.writeMessage(501, "Failed to parse parameters") + return + } + length, err := strconv.Atoi(moduleParams[1]) + if err != nil { + _, _ = conn.writeMessage(501, "Failed to parse parameters") + return + } + path := conn.buildPath(params[1]) + + if moduleName == mode.PartialFileTransport { + + bytes, data, err := conn.driver.GetFile(path, int64(offset)) + if err != nil { + _, _ = conn.writeMessage(551, "File not available") + return + } + + if length > int(bytes) { + length = int(bytes) + } + + buffer := make([]byte, length) + n, err := data.Read(buffer) + if n != length || err != nil { + _, _ = conn.writeMessage(551, "Error reading file") + return + } + + defer data.Close() + + _, _ = conn.writeMessage(150, fmt.Sprintf("Data transfer starting %v bytes", bytes)) + + conn.sendOutofbandData(buffer) + } else { + _, _ = conn.writeMessage(502, "Only PFT supported") + } +} diff --git a/ftpd/internal/core/cmd_test.go b/ftpd/internal/core/cmd_test.go new file mode 100644 index 00000000..cbdb093d --- /dev/null +++ b/ftpd/internal/core/cmd_test.go @@ -0,0 +1,34 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package core + +import "testing" + +func TestParseListParam(t *testing.T) { + var paramTests = []struct { + param string // input + expected string // expected result + }{ + {".", "."}, + {"-la", ""}, + {"-al", ""}, + {"rclone-test-qumelah4himezac1bogajow0", "rclone-test-qumelah4himezac1bogajow0"}, + {"-la rclone-test-qumelah4himezac1bogajow0", "rclone-test-qumelah4himezac1bogajow0"}, + {"-al rclone-test-qumelah4himezac1bogajow0", "rclone-test-qumelah4himezac1bogajow0"}, + {"rclone-test-goximif1kinarez5fakayuw7/new_name/sub_new_name", "rclone-test-goximif1kinarez5fakayuw7/new_name/sub_new_name"}, + {"rclone-test-qumelah4himezac1bogajow0/hello? sausage", "rclone-test-qumelah4himezac1bogajow0/hello? sausage"}, + {"rclone-test-qumelah4himezac1bogajow0/hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠", "rclone-test-qumelah4himezac1bogajow0/hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠"}, + {"rclone-test-qumelah4himezac1bogajow0/hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠/z.txt", "rclone-test-qumelah4himezac1bogajow0/hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠/z.txt"}, + {"rclone-test-qumelah4himezac1bogajow0/piped data.txt", "rclone-test-qumelah4himezac1bogajow0/piped data.txt"}, + {"rclone-test-qumelah4himezac1bogajow0/not found.txt", "rclone-test-qumelah4himezac1bogajow0/not found.txt"}, + } + + for _, tt := range paramTests { + path := parseListParam(tt.param) + if path != tt.expected { + t.Errorf("parseListParam(%s): expected %s, actual %s", tt.param, tt.expected, path) + } + } +} diff --git a/ftpd/internal/core/conn.go b/ftpd/internal/core/conn.go new file mode 100644 index 00000000..788836c3 --- /dev/null +++ b/ftpd/internal/core/conn.go @@ -0,0 +1,247 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// +// Copyright 2021 ETH Zurich modifications to add anonymous authentication + +package core + +import ( + "bufio" + crypto "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log" + "net" + "path/filepath" + "strconv" + "strings" + + "github.com/netsec-ethz/scion-apps/ftpd/internal/logger" + "github.com/netsec-ethz/scion-apps/internal/ftp/socket" +) + +const ( + defaultWelcomeMessage = "Welcome to the Go FTP Server" + listenerRetries = 10 +) + +type Conn struct { + conn net.Conn + controlReader *bufio.Reader + controlWriter *bufio.Writer + dataConn net.Conn + driver Driver + auth Auth + logger logger.Logger + server *Server + sessionID string + namePrefix string + reqUser string + user string + renameFrom string + lastFilePos int64 + appendData bool + closed bool + mode byte + parallelism int + blockSize int +} + +func (conn *Conn) LoginUser() string { + return conn.user +} + +func (conn *Conn) IsLogin() bool { + return len(conn.user) > 0 +} + +func (conn *Conn) NewListener() (*socket.Listener, error) { + + var err error + var listener *socket.Listener + + for i := 0; i < listenerRetries; i++ { + + listener, err = socket.ListenPort(0, conn.server.Certificate) + if err == nil { + break + } + } + + return listener, err +} + +// returns a random 20 char string that can be used as a unique session ID +func newSessionID() string { + hash := sha256.New() + _, err := io.CopyN(hash, crypto.Reader, 50) + if err != nil { + return "????????????????????" + } + md := hash.Sum(nil) + mdStr := hex.EncodeToString(md) + return mdStr[0:20] +} + +// Serve starts an endless loop that reads FTP commands from the scionftp and +// responds appropriately. terminated is a channel that will receive a true +// message when the connection closes. This loop will be running inside a +// goroutine, so use this channel to be notified when the connection can be +// cleaned up. +func (conn *Conn) Serve() { + conn.logger.Print(conn.sessionID, "Connection Established") + // send welcome + _, err := conn.writeMessage(220, conn.server.WelcomeMessage) + if err != nil { + conn.logger.Print(conn.sessionID, fmt.Sprint("write error:", err)) + } + // read commands + for { + line, err := conn.controlReader.ReadString('\n') + if err != nil { + if err != io.EOF { + conn.logger.Print(conn.sessionID, fmt.Sprint("read error:", err)) + } + + break + } + conn.receiveLine(line) + // QUIT command closes connection, break to avoid error on reading from + // closed socket + if conn.closed { + break + } + } + conn.Close() + conn.logger.Print(conn.sessionID, "Connection Terminated") +} + +// Close will manually close this connection, even if the scionftp isn't ready. +func (conn *Conn) Close() { + _ = conn.conn.Close() + conn.closed = true + if conn.dataConn != nil { + _ = conn.dataConn.Close() + conn.dataConn = nil + } +} + +// receiveLine accepts a single line FTP command and co-ordinates an +// appropriate response. +func (conn *Conn) receiveLine(line string) { + command, param := conn.parseLine(line) + conn.logger.PrintCommand(conn.sessionID, command, param) + cmdObj := commands[strings.ToUpper(command)] + if cmdObj == nil { + _, _ = conn.writeMessage(500, "Command not found") + return + } + if cmdObj.RequireParam() && param == "" { + _, _ = conn.writeMessage(553, "action aborted, required param missing") + } else if cmdObj.RequireAuth() && !conn.server.Auth.IsAuthorized(conn.user) { + _, _ = conn.writeMessage(530, "not logged in") + } else { + cmdObj.Execute(conn, param) + } +} + +func (conn *Conn) parseLine(line string) (string, string) { + params := strings.SplitN(strings.Trim(line, "\r\n"), " ", 2) + if len(params) == 1 { + return params[0], "" + } + return params[0], strings.TrimSpace(params[1]) +} + +// writeMessage will send a standard FTP response back to the scionftp. +func (conn *Conn) writeMessage(code int, message string) (wrote int, err error) { + conn.logger.PrintResponse(conn.sessionID, code, message) + line := fmt.Sprintf("%d %s\r\n", code, message) + wrote, err = conn.controlWriter.WriteString(line) + if err == nil { + err = conn.controlWriter.Flush() + } + return +} + +func (conn *Conn) writeOrLog(code int, message string) { + _, err := conn.writeMessage(code, message) + if err != nil { + log.Printf("Could not write message (%d %s): %s", code, message, err) + } +} + +// writeMessage will send a standard FTP response back to the scionftp. +func (conn *Conn) writeMessageMultiline(code int, message string) (wrote int, err error) { + conn.logger.PrintResponse(conn.sessionID, code, message) + line := fmt.Sprintf("%d-%s\r\n%d END\r\n", code, message, code) + wrote, err = conn.controlWriter.WriteString(line) + if err == nil { + err = conn.controlWriter.Flush() + } + return +} + +// buildPath takes a scionftp supplied path or filename and generates a safe +// absolute path within their account sandbox. +// +// buildpath("/") +// => "/" +// buildpath("one.txt") +// => "/one.txt" +// buildpath("/files/two.txt") +// => "/files/two.txt" +// buildpath("files/two.txt") +// => "/files/two.txt" +// buildpath("/../../../../etc/passwd") +// => "/etc/passwd" +// +// The driver implementation is responsible for deciding how to treat this path. +// Obviously they MUST NOT just read the path off disk. The probably want to +// prefix the path with something to scope the users access to a sandbox. +func (conn *Conn) buildPath(filename string) (fullPath string) { + if len(filename) > 0 && filename[0:1] == "/" { + fullPath = filepath.Clean(filename) + } else if len(filename) > 0 && filename != "-a" { + fullPath = filepath.Clean(conn.namePrefix + "/" + filename) + } else { + fullPath = filepath.Clean(conn.namePrefix) + } + fullPath = strings.Replace(fullPath, "//", "/", -1) + fullPath = strings.Replace(fullPath, string(filepath.Separator), "/", -1) + return +} + +// sendOutofbandData will send a string to the scionftp via the currently open +// data socket. Assumes the socket is open and ready to be used. +func (conn *Conn) sendOutofbandData(data []byte) { + bytes := len(data) + if conn.dataConn != nil { + _, _ = conn.dataConn.Write(data) + _ = conn.dataConn.Close() + conn.dataConn = nil + } + message := "Closing data connection, sent " + strconv.Itoa(bytes) + " bytes" + _, _ = conn.writeMessage(226, message) +} + +func (conn *Conn) sendOutofBandDataWriter(data io.ReadCloser) error { + conn.lastFilePos = 0 + bytes, err := io.Copy(conn.dataConn, data) + if err != nil { + err = conn.dataConn.Close() + conn.dataConn = nil + return err + } + message := "Closing data connection, sent " + strconv.Itoa(int(bytes)) + " bytes" + _, err = conn.writeMessage(226, message) + if err == nil { + err = conn.dataConn.Close() + } + conn.dataConn = nil + + return err +} diff --git a/ftpd/internal/core/conn_test.go b/ftpd/internal/core/conn_test.go new file mode 100644 index 00000000..a2fe8755 --- /dev/null +++ b/ftpd/internal/core/conn_test.go @@ -0,0 +1,34 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package core + +import ( + "testing" +) + +func TestConnBuildPath(t *testing.T) { + c := &Conn{ + namePrefix: "", + } + var pathtests = []struct { + in string + out string + }{ + {"/", "/"}, + {"one.txt", "/one.txt"}, + {"/files/two.txt", "/files/two.txt"}, + {"files/two.txt", "/files/two.txt"}, + {"/../../../../etc/passwd", "/etc/passwd"}, + {"rclone-test-roxarey8facabob5tuwetet4/hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠/z.txt", "/rclone-test-roxarey8facabob5tuwetet4/hello? sausage/êé/Hello, 世界/ \" ' @ < > & ? + ≠/z.txt"}, + } + for _, tt := range pathtests { + t.Run(tt.in, func(t *testing.T) { + s := c.buildPath(tt.in) + if s != tt.out { + t.Errorf("got %q, want %q", s, tt.out) + } + }) + } +} diff --git a/ftpd/internal/core/doc.go b/ftpd/internal/core/doc.go new file mode 100644 index 00000000..47ba6bb0 --- /dev/null +++ b/ftpd/internal/core/doc.go @@ -0,0 +1,14 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +/* +http://tools.ietf.org/html/rfc959 + +http://www.faqs.org/rfcs/rfc2389.html +http://www.faqs.org/rfcs/rfc959.html + +http://tools.ietf.org/html/rfc2428 +*/ + +package core diff --git a/ftpd/internal/core/driver.go b/ftpd/internal/core/driver.go new file mode 100644 index 00000000..7ca4099b --- /dev/null +++ b/ftpd/internal/core/driver.go @@ -0,0 +1,77 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// +// Copyright 2020-2021 ETH Zurich modifications to add support for SCION + +package core + +import ( + "fmt" + "io" +) + +var ErrNoFileSystem = fmt.Errorf("driver is not backed by a filesystem") + +// DriverFactory is a driver factory to create driver. For each scionftp that connects to the server, a new FTPDriver is required. +// Create an implementation if this interface and provide it to FTPServer. +type DriverFactory interface { + NewDriver() (Driver, error) +} + +// Driver is an interface that you will create an implementation that speaks to your +// chosen persistence layer. graval will create a new instance of your +// driver for each scionftp that connects and delegate to it as required. +type Driver interface { + // Init init + Init(*Conn) + + // params - a file path + // returns - a time indicating when the requested path was last modified + // - an error if the file doesn't exist or the user lacks + // permissions + Stat(string) (FileInfo, error) + + // params - path + // returns - true if the current user is permitted to change to the + // requested path + ChangeDir(string) error + + // params - path, function on file or subdir found + // returns - error + // path + ListDir(string, func(FileInfo) error) error + + // params - path + // returns - nil if the directory was deleted or any error encountered + DeleteDir(string) error + + // params - path + // returns - nil if the file was deleted or any error encountered + DeleteFile(string) error + + // params - from_path, to_path + // returns - nil if the file was renamed or any error encountered + Rename(string, string) error + + // params - path + // returns - nil if the new directory was created or any error encountered + MakeDir(string) error + + // params - path + // returns - a string containing the file data to send to the scionftp + GetFile(string, int64) (int64, io.ReadCloser, error) + + // params - destination path, an io.Reader containing the file data + // returns - the number of bytes written and the first error encountered while writing, if any. + PutFile(string, io.Reader, bool) (int64, error) + + // This function is only required to support the Hercules subsystem. If the driver is not backed by a filesystem, + // this function should return a dummy value and IsFileSystem() must return false. + // params - path + // returns - valid path in the filesystem to use with the Hercules subsystem + RealPath(path string) string + + // returns - whether the driver is backed by a filesystem + IsFileSystem() bool +} diff --git a/ftpd/internal/core/fileinfo.go b/ftpd/internal/core/fileinfo.go new file mode 100644 index 00000000..d75a5c08 --- /dev/null +++ b/ftpd/internal/core/fileinfo.go @@ -0,0 +1,14 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package core + +import "os" + +type FileInfo interface { + os.FileInfo + + Owner() string + Group() string +} diff --git a/ftpd/internal/core/listformatter.go b/ftpd/internal/core/listformatter.go new file mode 100644 index 00000000..33f4222b --- /dev/null +++ b/ftpd/internal/core/listformatter.go @@ -0,0 +1,51 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// +// Copyright 2020-2021 ETH Zurich modifications to add support for SCION + +package core + +import ( + "bytes" + "fmt" + "strconv" + "strings" +) + +type listFormatter []FileInfo + +// Short returns a string that lists the collection of files by name only, +// one per line +func (formatter listFormatter) Short() []byte { + var buf bytes.Buffer + for _, file := range formatter { + fmt.Fprintf(&buf, "%s\r\n", file.Name()) + } + return buf.Bytes() +} + +// Detailed returns a string that lists the collection of files with extra +// detail, one per line +func (formatter listFormatter) Detailed() []byte { + var buf bytes.Buffer + for _, file := range formatter { + fmt.Fprint(&buf, file.Mode().String()) + fmt.Fprintf(&buf, " 1 %s %s ", file.Owner(), file.Group()) + fmt.Fprint(&buf, lpad(strconv.FormatInt(file.Size(), 10), 12)) + fmt.Fprint(&buf, file.ModTime().Format(" Jan _2 15:04 ")) + fmt.Fprintf(&buf, "%s\r\n", file.Name()) + } + return buf.Bytes() +} + +func lpad(input string, length int) (result string) { + if len(input) < length { + result = strings.Repeat(" ", length-len(input)) + input + } else if len(input) == length { + result = input + } else { + result = input[0:length] + } + return +} diff --git a/ftpd/internal/core/lock.go b/ftpd/internal/core/lock.go new file mode 100644 index 00000000..abdd772f --- /dev/null +++ b/ftpd/internal/core/lock.go @@ -0,0 +1,36 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import "time" + +type lock chan struct{} + +func makeLock() lock { + return make(lock, 1) +} + +func (lock lock) unlock() { + <-lock +} + +func (lock lock) tryLockTimeout(timeout time.Duration) bool { + select { + case lock <- struct{}{}: + return true + case <-time.After(timeout): + return false + } +} diff --git a/ftpd/internal/core/perm.go b/ftpd/internal/core/perm.go new file mode 100644 index 00000000..312ec301 --- /dev/null +++ b/ftpd/internal/core/perm.go @@ -0,0 +1,52 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package core + +import "os" + +type Perm interface { + GetOwner(string) (string, error) + GetGroup(string) (string, error) + GetMode(string) (os.FileMode, error) + + ChOwner(string, string) error + ChGroup(string, string) error + ChMode(string, os.FileMode) error +} + +type SimplePerm struct { + owner, group string +} + +func NewSimplePerm(owner, group string) *SimplePerm { + return &SimplePerm{ + owner: owner, + group: group, + } +} + +func (s *SimplePerm) GetOwner(string) (string, error) { + return s.owner, nil +} + +func (s *SimplePerm) GetGroup(string) (string, error) { + return s.group, nil +} + +func (s *SimplePerm) GetMode(string) (os.FileMode, error) { + return os.ModePerm, nil +} + +func (s *SimplePerm) ChOwner(string, string) error { + return nil +} + +func (s *SimplePerm) ChGroup(string, string) error { + return nil +} + +func (s *SimplePerm) ChMode(string, os.FileMode) error { + return nil +} diff --git a/ftpd/internal/core/server.go b/ftpd/internal/core/server.go new file mode 100644 index 00000000..1f8a51c5 --- /dev/null +++ b/ftpd/internal/core/server.go @@ -0,0 +1,226 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// +// Copyright 2020-2021 ETH Zurich modifications to add support for SCION + +package core + +import ( + "bufio" + "context" + "crypto/tls" + "errors" + "fmt" + "net" + + "github.com/netsec-ethz/scion-apps/ftpd/internal/logger" + "github.com/netsec-ethz/scion-apps/internal/ftp/socket" +) + +// ServerOpts contains parameters for ftpd.NewServer() +type Opts struct { + // The factory that will be used to create a new FTPDriver instance for + // each scionftp connection. This is a mandatory option. + Factory DriverFactory + + Auth Auth + + // Server Name, Default is Go Ftp Server + Name string + + // The port that the FTP should listen on. Optional, defaults to 2121. In + // a production environment you will probably want to change this to 21. + Port uint16 + + WelcomeMessage string + + // A logger implementation, if nil the StdLogger is used + Logger logger.Logger + + Certificate *tls.Certificate + + // Hercules binary for RETR_HERCULES feature + HerculesBinary string + HerculesConfig *string + RootPath string +} + +// Server is the root of your FTP application. You should instantiate one +// of these and call ListenAndServe() to start accepting cilent connections. +// +// Always use the NewServer() method to create a new Server. +type Server struct { + *Opts + logger logger.Logger + listener *socket.Listener + ctx context.Context + cancel context.CancelFunc + feats string + herculesLock lock +} + +// ErrServerClosed is returned by ListenAndServe() or Serve() when a shutdown +// was requested. +var ErrServerClosed = errors.New("ftp: Server closed") + +// serverOptsWithDefaults copies an ServerOpts struct into a new struct, +// then adds any default values that are missing and returns the new data. +func serverOptsWithDefaults(opts *Opts) *Opts { + var newOpts Opts + if opts == nil { + opts = &Opts{} + } + if opts.Port == 0 { + newOpts.Port = 2121 + } else { + newOpts.Port = opts.Port + } + newOpts.Factory = opts.Factory + if opts.Name == "" { + newOpts.Name = "Go FTP Server" + } else { + newOpts.Name = opts.Name + } + + if opts.WelcomeMessage == "" { + newOpts.WelcomeMessage = defaultWelcomeMessage + } else { + newOpts.WelcomeMessage = opts.WelcomeMessage + } + + if opts.Auth != nil { + newOpts.Auth = opts.Auth + } + + newOpts.Logger = &logger.StdLogger{} + if opts.Logger != nil { + newOpts.Logger = opts.Logger + } + + newOpts.Certificate = opts.Certificate + newOpts.HerculesBinary = opts.HerculesBinary + newOpts.HerculesConfig = opts.HerculesConfig + newOpts.RootPath = opts.RootPath + + return &newOpts +} + +// NewServer initialises a new FTP server. Configuration options are provided +// via an instance of ServerOpts. Calling this function in your code will +// probably look something like this: +// +// factory := &MyDriverFactory{} +// server := ftpd.NewServer(&server.ServerOpts{ Factory: factory }) +// +// or: +// +// factory := &MyDriverFactory{} +// opts := &server.ServerOpts{ +// Factory: factory, +// Port: 2000, +// Hostname: "127.0.0.1", +// } +// server := ftpd.NewServer(opts) +// +func NewServer(opts *Opts) *Server { + opts = serverOptsWithDefaults(opts) + s := new(Server) + s.Opts = opts + s.logger = opts.Logger + s.herculesLock = makeLock() + return s +} + +// NewConn constructs a new object that will handle the FTP protocol over +// an active net.TCPConn. The TCP connection should already be open before +// it is handed to this functions. driver is an instance of FTPDriver that +// will handle all auth and persistence details. +func (server *Server) newConn(tcpConn net.Conn, driver Driver) *Conn { + c := new(Conn) + c.namePrefix = "/" + c.conn = tcpConn + c.controlReader = bufio.NewReader(tcpConn) + c.controlWriter = bufio.NewWriter(tcpConn) + c.driver = driver + c.auth = server.Auth + c.server = server + c.sessionID = newSessionID() + c.logger = server.logger + + driver.Init(c) + return c +} + +// ListenAndServe asks a new Server to begin accepting scionftp connections. It +// accepts no arguments - all configuration is provided via the NewServer +// function. +// +// If the server fails to start for any reason, an error will be returned. Common +// errors are trying to bind to a privileged port or something else is already +// listening on the same port. +// +func (server *Server) ListenAndServe() error { + var listener *socket.Listener + var err error + var curFeats = featCmds + + listener, err = socket.ListenPort(server.Port, server.Certificate) + if err != nil { + return err + } + + if server.HerculesBinary != "" { + curFeats += " HERCULES\n" + } + server.feats = fmt.Sprintf(feats, curFeats) + + sessionID := "" + server.logger.Printf(sessionID, "%s listening on %d", server.Name, server.Port) + + return server.Serve(listener) +} + +// Serve accepts connections on a given net.Listener and handles each +// request in a new goroutine. +// +func (server *Server) Serve(l *socket.Listener) error { + server.listener = l + server.ctx, server.cancel = context.WithCancel(context.Background()) + sessionID := "" + for { + conn, err := server.listener.Accept() + if err != nil { + select { + case <-server.ctx.Done(): + return ErrServerClosed + default: + } + server.logger.Printf(sessionID, "listening error: %v", err) + if ne, ok := err.(net.Error); ok && ne.Temporary() { + continue + } + return err + } + driver, err := server.Factory.NewDriver() + if err != nil { + server.logger.Printf(sessionID, "Error creating driver, aborting scionftp connection: %v", err) + conn.Close() + } else { + ftpConn := server.newConn(conn, driver) + go ftpConn.Serve() + } + } +} + +// Shutdown will gracefully stop a server. Already connected clients will retain their connections +func (server *Server) Shutdown() error { + if server.cancel != nil { + server.cancel() + } + if server.listener != nil { + return server.listener.Close() + } + // server wasn't even started + return nil +} diff --git a/ftpd/internal/core/server_test.go b/ftpd/internal/core/server_test.go new file mode 100644 index 00000000..99ea1a66 --- /dev/null +++ b/ftpd/internal/core/server_test.go @@ -0,0 +1,170 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// +// Copyright 2020-2021 ETH Zurich modifications to add support for SCION + +package core_test + +import ( + "io/ioutil" + "net" + "os" + "strings" + "testing" + "time" + + filedriver "gitea.com/goftp/file-driver" + "github.com/jlaffaye/ftp" + "github.com/stretchr/testify/assert" + "goftp.io/server" +) + +func runServer(t *testing.T, execute func()) { + assert.NoError(t, os.MkdirAll("./testdata", os.ModePerm)) + + var perm = server.NewSimplePerm("test", "test") + opt := &server.ServerOpts{ + Name: "test ftpd", + Factory: &filedriver.FileDriverFactory{ + RootPath: "./testdata", + Perm: perm, + }, + Port: 2121, + Auth: &server.SimpleAuth{ + Name: "admin", + Password: "admin", + }, + Logger: new(server.DiscardLogger), + } + + s := server.NewServer(opt) + go func() { + err := s.ListenAndServe() + assert.EqualError(t, err, server.ErrServerClosed.Error()) + }() + + execute() + + assert.NoError(t, s.Shutdown()) +} + +func TestConnect(t *testing.T) { + runServer(t, func() { + // Give server 0.5 seconds to get to the listening state + timeout := time.NewTimer(time.Millisecond * 500) + for { + f, err := ftp.Connect("localhost:2121") + if err != nil && len(timeout.C) == 0 { // Retry errors + continue + } + assert.NoError(t, err) + + assert.NoError(t, f.Login("admin", "admin")) + assert.Error(t, f.Login("admin", "")) + + var content = `test` + assert.NoError(t, f.Stor("server_test.go", strings.NewReader(content))) + + names, err := f.NameList("/") + assert.NoError(t, err) + assert.EqualValues(t, 1, len(names)) + assert.EqualValues(t, "server_test.go", names[0]) + + bs, err := ioutil.ReadFile("./testdata/server_test.go") + assert.NoError(t, err) + assert.EqualValues(t, content, string(bs)) + + entries, err := f.List("/") + assert.NoError(t, err) + assert.EqualValues(t, 1, len(entries)) + assert.EqualValues(t, "server_test.go", entries[0].Name) + assert.EqualValues(t, 4, entries[0].Size) + assert.EqualValues(t, ftp.EntryTypeFile, entries[0].Type) + + curDir, err := f.CurrentDir() + assert.NoError(t, err) + assert.EqualValues(t, "/", curDir) + + size, err := f.FileSize("/server_test.go") + assert.NoError(t, err) + assert.EqualValues(t, 4, size) + + /*resp, err := f.RetrFrom("/server_test.go", 0) + assert.NoError(t, err) + var buf []byte + l, err := resp.Read(buf) + assert.NoError(t, err) + assert.EqualValues(t, 4, l) + assert.EqualValues(t, 4, len(buf)) + assert.EqualValues(t, content, string(buf))*/ + + err = f.Rename("/server_test.go", "/server.test.go") + assert.NoError(t, err) + + err = f.MakeDir("/src") + assert.NoError(t, err) + + err = f.Delete("/server.test.go") + assert.NoError(t, err) + + err = f.RemoveDir("/src") + assert.NoError(t, err) + + err = f.Quit() + assert.NoError(t, err) + + break + } + }) +} + +func TestServe(t *testing.T) { + assert.NoError(t, os.MkdirAll("./testdata", os.ModePerm)) + + var perm = server.NewSimplePerm("test", "test") + + // Server options without hostname or port + opt := &server.ServerOpts{ + Name: "test ftpd", + Factory: &filedriver.FileDriverFactory{ + RootPath: "./testdata", + Perm: perm, + }, + Auth: &server.SimpleAuth{ + Name: "admin", + Password: "admin", + }, + Logger: new(server.DiscardLogger), + } + + // Start the listener + l, err := net.Listen("tcp", ":2121") + assert.NoError(t, err) + + // Start the server using the listener + s := server.NewServer(opt) + go func() { + err := s.Serve(l) + assert.EqualError(t, err, server.ErrServerClosed.Error()) + }() + + // Give server 0.5 seconds to get to the listening state + timeout := time.NewTimer(time.Millisecond * 500) + for { + f, err := ftp.Connect("localhost:2121") + if err != nil && len(timeout.C) == 0 { // Retry errors + continue + } + assert.NoError(t, err) + + assert.NoError(t, f.Login("admin", "admin")) + assert.Error(t, f.Login("admin", "")) + + err = f.Quit() + assert.NoError(t, err) + break + } + + assert.NoError(t, s.Shutdown()) +} diff --git a/ftpd/internal/driver/file/driver.go b/ftpd/internal/driver/file/driver.go new file mode 100644 index 00000000..6279bb58 --- /dev/null +++ b/ftpd/internal/driver/file/driver.go @@ -0,0 +1,254 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package filedriver + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/netsec-ethz/scion-apps/ftpd/internal/core" +) + +var _ core.Driver = &FileDriver{} + +type FileDriver struct { + RootPath string + core.Perm +} + +func (driver *FileDriver) Init(*core.Conn) { + // +} + +type FileInfo struct { + os.FileInfo + + mode os.FileMode + owner string + group string +} + +func (f *FileInfo) Mode() os.FileMode { + return f.mode +} + +func (f *FileInfo) Owner() string { + return f.owner +} + +func (f *FileInfo) Group() string { + return f.group +} + +func (driver *FileDriver) RealPath(path string) string { + paths := strings.Split(path, "/") + return filepath.Join(append([]string{driver.RootPath}, paths...)...) +} + +func (driver *FileDriver) IsFileSystem() bool { + return true +} + +func (driver *FileDriver) ChangeDir(path string) error { + rPath := driver.RealPath(path) + f, err := os.Lstat(rPath) + if err != nil { + return err + } + if f.IsDir() { + return nil + } + return errors.New("not a directory") +} + +func (driver *FileDriver) Stat(path string) (core.FileInfo, error) { + basepath := driver.RealPath(path) + rPath, err := filepath.Abs(basepath) + if err != nil { + return nil, err + } + f, err := os.Lstat(rPath) + if err != nil { + return nil, err + } + mode, err := driver.Perm.GetMode(path) + if err != nil { + return nil, err + } + if f.IsDir() { + mode |= os.ModeDir + } + owner, err := driver.Perm.GetOwner(path) + if err != nil { + return nil, err + } + group, err := driver.Perm.GetGroup(path) + if err != nil { + return nil, err + } + return &FileInfo{f, mode, owner, group}, nil +} + +func (driver *FileDriver) ListDir(path string, callback func(core.FileInfo) error) error { + basepath := driver.RealPath(path) + return filepath.Walk(basepath, func(f string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rPath, _ := filepath.Rel(basepath, f) + if rPath == info.Name() { + mode, err := driver.Perm.GetMode(rPath) + if err != nil { + return err + } + if info.IsDir() { + mode |= os.ModeDir + } + owner, err := driver.Perm.GetOwner(rPath) + if err != nil { + return err + } + group, err := driver.Perm.GetGroup(rPath) + if err != nil { + return err + } + err = callback(&FileInfo{info, mode, owner, group}) + if err != nil { + return err + } + if info.IsDir() { + return filepath.SkipDir + } + } + return nil + }) +} + +func (driver *FileDriver) DeleteDir(path string) error { + rPath := driver.RealPath(path) + f, err := os.Lstat(rPath) + if err != nil { + return err + } + if f.IsDir() { + return os.Remove(rPath) + } + return errors.New("not a directory") +} + +func (driver *FileDriver) DeleteFile(path string) error { + rPath := driver.RealPath(path) + f, err := os.Lstat(rPath) + if err != nil { + return err + } + if !f.IsDir() { + return os.Remove(rPath) + } + return errors.New("not a file") +} + +func (driver *FileDriver) Rename(fromPath string, toPath string) error { + oldPath := driver.RealPath(fromPath) + newPath := driver.RealPath(toPath) + return os.Rename(oldPath, newPath) +} + +func (driver *FileDriver) MakeDir(path string) error { + rPath := driver.RealPath(path) + return os.MkdirAll(rPath, os.ModePerm) +} + +func (driver *FileDriver) GetFile(path string, offset int64) (int64, io.ReadCloser, error) { + rPath := driver.RealPath(path) + f, err := os.Open(rPath) + if err != nil { + return 0, nil, err + } + + info, err := f.Stat() + if err != nil { + return 0, nil, err + } + + _, err = f.Seek(offset, io.SeekStart) + if err != nil { + return 0, nil, err + } + + return info.Size(), f, nil +} + +func (driver *FileDriver) PutFile(destPath string, data io.Reader, appendData bool) (int64, error) { + rPath := driver.RealPath(destPath) + var isExist bool + f, err := os.Lstat(rPath) + if err == nil { + isExist = true + if f.IsDir() { + return 0, errors.New("a dir has the same name") + } + } else { + if os.IsNotExist(err) { + isExist = false + } else { + return 0, errors.New(fmt.Sprintln("put File error:", err)) + } + } + + if appendData && !isExist { + appendData = false + } + + if !appendData { + if isExist { + err = os.Remove(rPath) + if err != nil { + return 0, err + } + } + f, err := os.Create(rPath) + if err != nil { + return 0, err + } + defer f.Close() + bytes, err := io.Copy(f, data) + if err != nil { + return 0, err + } + return bytes, nil + } + + of, err := os.OpenFile(rPath, os.O_APPEND|os.O_RDWR, 0660) + if err != nil { + return 0, err + } + defer of.Close() + + _, err = of.Seek(0, io.SeekEnd) + if err != nil { + return 0, err + } + + bytes, err := io.Copy(of, data) + if err != nil { + return 0, err + } + + return bytes, nil +} + +type FileDriverFactory struct { + RootPath string + core.Perm +} + +func (factory *FileDriverFactory) NewDriver() (core.Driver, error) { + return &FileDriver{factory.RootPath, factory.Perm}, nil +} diff --git a/ftpd/internal/logger/logger.go b/ftpd/internal/logger/logger.go new file mode 100644 index 00000000..fd208230 --- /dev/null +++ b/ftpd/internal/logger/logger.go @@ -0,0 +1,50 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// +// Copyright 2020 ETH Zurich modifications to pass scion-apps linting + +package logger + +import ( + "fmt" + "log" +) + +type Logger interface { + Print(sessionID string, message interface{}) + Printf(sessionID string, format string, v ...interface{}) + PrintCommand(sessionID string, command string, params string) + PrintResponse(sessionID string, code int, message string) +} + +// Use an instance of this to log in a standard format +type StdLogger struct{} + +func (logger *StdLogger) Print(sessionID string, message interface{}) { + log.Printf("%s %s", sessionID, message) +} + +func (logger *StdLogger) Printf(sessionID string, format string, v ...interface{}) { + logger.Print(sessionID, fmt.Sprintf(format, v...)) +} + +func (logger *StdLogger) PrintCommand(sessionID string, command string, params string) { + if command == "PASS" { + log.Printf("%s > PASS ****", sessionID) + } else { + log.Printf("%s > %s %s", sessionID, command, params) + } +} + +func (logger *StdLogger) PrintResponse(sessionID string, code int, message string) { + log.Printf("%s < %d %s", sessionID, code, message) +} + +// Silent logger, produces no output +type DiscardLogger struct{} + +func (logger *DiscardLogger) Print(sessionID string, message interface{}) {} +func (logger *DiscardLogger) Printf(sessionID string, format string, v ...interface{}) {} +func (logger *DiscardLogger) PrintCommand(sessionID string, command string, params string) {} +func (logger *DiscardLogger) PrintResponse(sessionID string, code int, message string) {} diff --git a/ftpd/main.go b/ftpd/main.go new file mode 100644 index 00000000..3e677524 --- /dev/null +++ b/ftpd/main.go @@ -0,0 +1,73 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// +// Modifications 2019 Elwin Stephan to make it compatible to SCION and +// introduce parts of the GridFTP extension +// +// Copyright 2020-2021 ETH Zurich modifications to add support for SCION +package main + +import ( + "flag" + "log" + + "github.com/netsec-ethz/scion-apps/ftpd/internal/core" + driver "github.com/netsec-ethz/scion-apps/ftpd/internal/driver/file" + libhercules "github.com/netsec-ethz/scion-apps/internal/ftp/hercules" + "github.com/netsec-ethz/scion-apps/pkg/appnet/appquic" +) + +func main() { + var ( + root = flag.String("root", "", "Root directory to serve") + user = flag.String("user", "", "Username for login (omit for public access)") + pass = flag.String("pass", "", "Password for login (omit for public access)") + port = flag.Uint("port", 2121, "Port") + hercules = flag.String("hercules", "", "Enable Hercules mode using the Hercules binary specified\nIn Hercules mode, scionFTP checks the following directories for Hercules config files: ., /etc, /etc/scion-ftp") + ) + flag.Parse() + if *root == "" { + log.Fatalf("Please set a root to serve with -root") + } + + factory := &driver.FileDriverFactory{ + RootPath: *root, + Perm: core.NewSimplePerm("user", "group"), + } + + certs := appquic.GetDummyTLSCerts() + + var auth core.Auth + if *user == "" && *pass == "" { + log.Printf("Anonymous FTP") + auth = &core.AnonymousAuth{} + } else { + log.Printf("Username %v, Password %v", *user, *pass) + auth = &core.SimpleAuth{Name: *user, Password: *pass} + } + + herculesConfig, err := libhercules.ResolveConfig() + if err != nil { + log.Printf("hercules.ResolveConfig: %s", err) + } + if herculesConfig != nil { + log.Printf("In Hercules mode, using configuration at %s", *herculesConfig) + } + + opts := &core.Opts{ + Factory: factory, + Port: uint16(*port), + Auth: auth, + Certificate: &certs[0], + HerculesBinary: *hercules, + HerculesConfig: herculesConfig, + RootPath: *root, + } + + srv := core.NewServer(opts) + err = srv.ListenAndServe() + if err != nil { + log.Fatal("Error starting server:", err) + } +} diff --git a/go.mod b/go.mod index b320e46d..10c86111 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,12 @@ module github.com/netsec-ethz/scion-apps go 1.14 require ( + gitea.com/goftp/file-driver v0.0.0-20190812052443-efcdcba68b34 github.com/BurntSushi/toml v0.3.1 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect github.com/bclicn/color v0.0.0-20180711051946-108f2023dc84 github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec + github.com/jlaffaye/ftp v0.0.0-20201021201046-0de5c29d4555 github.com/kormat/fmt15 v0.0.0-20181112140556-ee69fecb2656 github.com/kr/pty v1.1.8 github.com/lucas-clemente/quic-go v0.19.2 @@ -15,6 +17,8 @@ require ( github.com/netsec-ethz/rains v0.2.0 github.com/scionproto/scion v0.6.0 github.com/smartystreets/goconvey v1.6.4 + github.com/stretchr/testify v1.6.1 + goftp.io/server v0.4.0 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 gopkg.in/alecthomas/kingpin.v2 v2.2.6 diff --git a/go.sum b/go.sum index 2a59bfd0..b42de16d 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,9 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +gitea.com/goftp/file-driver v0.0.0-20190712091345-f79c2ed973f8/go.mod h1:ghdogu0Da3rwYCSJ20JPgTiMcDpzeRbzvuFIOOW3G7w= +gitea.com/goftp/file-driver v0.0.0-20190812052443-efcdcba68b34 h1:3wshUWDKHcy8hrNafCS4rtuAdON2KYsuznc05zdHTrQ= +gitea.com/goftp/file-driver v0.0.0-20190812052443-efcdcba68b34/go.mod h1:6+f1gclV97PmaVmE4YJbH3KIKnl+r3/HWR0zD/z1CG4= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -99,11 +102,9 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -120,6 +121,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goftp/file-driver v0.0.0-20180502053751-5d604a0fc0c9/go.mod h1:GpOj6zuVBG3Inr9qjEnuVTgBlk2lZ1S9DcoFiXWyKss= +github.com/goftp/server v0.0.0-20190304020633-eabccc535b5a/go.mod h1:k/SS6VWkxY7dHPhoMQ8IdRu8L4lQtmGbhyXGg+vCnXE= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -132,20 +135,17 @@ github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4er github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= @@ -153,7 +153,6 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= @@ -209,7 +208,6 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365 h1:ECW73yc9MY7935nNYXUkK7Dz17YuSUI9yqRqYS8aBww= @@ -219,6 +217,9 @@ github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec/go.mod h1:cO github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= +github.com/jlaffaye/ftp v0.0.0-20201021201046-0de5c29d4555 h1:bd2tFFziQpwjrRcj7seCELvu08uplHN7Fs5t2/9kQNE= +github.com/jlaffaye/ftp v0.0.0-20201021201046-0de5c29d4555/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -248,7 +249,6 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lucas-clemente/quic-go v0.17.3 h1:jMX/MmDNCljfisgMmPGUcBJ+zUh9w3d3ia4YJjYS3TM= github.com/lucas-clemente/quic-go v0.17.3/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE= github.com/lucas-clemente/quic-go v0.19.2 h1:w8BBYUx5Z+kNpeaOeQW/KzcNsKWhh4O6PeQhb0nURPg= github.com/lucas-clemente/quic-go v0.19.2/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0= @@ -256,24 +256,20 @@ github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/marten-seemann/qpack v0.1.0 h1:/0M7lkda/6mus9B8u34Asqm8ZhHAAt9Ho0vniNuVSVg= github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI= github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ= github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk= github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= github.com/marten-seemann/qtls-go1-15 v0.1.1 h1:LIH6K34bPVttyXnUWixk0bzH6/N07VxbSabxn5A5gZQ= github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -286,8 +282,13 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/minio-go/v6 v6.0.46 h1:waExJtO53xrnsNX//7cSc1h3478wqTryDx4RVD7o26I= +github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -320,14 +321,12 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -344,7 +343,6 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/patrickmn/go-cache v2.1.1-0.20180815053127-5633e0862627+incompatible h1:MUIwjEiAMYk8zkXXUQeb5itrXF+HpS2pfxNsA2a7AiY= github.com/patrickmn/go-cache v2.1.1-0.20180815053127-5633e0862627+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -374,7 +372,6 @@ github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83A github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= @@ -435,6 +432,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -459,10 +457,10 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/gocapability v0.0.0-20160928074757-e7cb7fa329f4/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= @@ -506,6 +504,10 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +goftp.io/server v0.0.0-20190712054601-1149070ae46b/go.mod h1:xreggPYu7ZuNe9PfbxiQca7bYGwU44IvlCCg3KzWJtQ= +goftp.io/server v0.0.0-20190812034929-9b3874d17690/go.mod h1:99FISrRpwKfaL4Ey/dX8N48WToveng/s2OXR5sJ3cnc= +goftp.io/server v0.4.0 h1:hqsVdwd1/l6QtYxD9pxca9mEAJYZ7+FPCnmeXKXHQNw= +goftp.io/server v0.4.0/go.mod h1:hFZeR656ErRt3ojMKt7H10vQ5nuWV1e0YeUTeorlR6k= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -513,12 +515,11 @@ golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -558,6 +559,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -601,17 +603,13 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -631,18 +629,14 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114 h1:DnSr2mCsxyCE6ZgIkmcWUQY2R5cH/6wL7eIxEmQOMSE= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f h1:7+Nz9MyPqt2qMCTvNiRy1G0zYfkB7UCa+ayT6uVvbyI= golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -688,7 +682,6 @@ google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyz google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= @@ -697,10 +690,11 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/d4l3k/messagediff.v1 v1.2.1 h1:70AthpjunwzUiarMHyED52mj9UwtAnE89l1Gmrt3EU0= gopkg.in/d4l3k/messagediff.v1 v1.2.1/go.mod h1:EUzikiKadqXWcD1AzJLagx0j/BeeWGtn++04Xniyg44= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -708,11 +702,12 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/ftp/hercules/utils.go b/internal/ftp/hercules/utils.go new file mode 100644 index 00000000..ff524417 --- /dev/null +++ b/internal/ftp/hercules/utils.go @@ -0,0 +1,175 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hercules + +import ( + "errors" + "fmt" + "net" + "os" + "os/exec" + "os/user" + "strconv" + + "github.com/scionproto/scion/go/lib/snet" + + "github.com/netsec-ethz/scion-apps/pkg/appnet" +) + +func findInterfaceName(localAddr net.IP) (string, error) { + ifaces, err := net.Interfaces() + if err != nil { + return "", fmt.Errorf("could not retrieve network interfaces: %s", err) + } + + for _, iface := range ifaces { + addrs, err := iface.Addrs() + if err != nil { + return "", fmt.Errorf("could not get interface addresses: %s", err) + } + + if iface.Flags&net.FlagUp == 0 { + continue // interface not up + } + + for _, addr := range addrs { + ip, ok := addr.(*net.IPNet) + if ok && ip.IP.To4() != nil && ip.IP.To4().Equal(localAddr) { + return iface.Name, nil + } + } + } + + return "", fmt.Errorf("could not find interface with address %s", localAddr) +} + +func prepareHerculesArgs(herculesBinary string, herculesConfig *string, localAddress *net.UDPAddr, offset int64) ([]string, error) { + iface, err := findInterfaceName(localAddress.IP) + if err != nil { + return nil, err + } + + lAddr := &snet.UDPAddr{ + IA: appnet.DefNetwork().IA, + Host: localAddress, + } + + args := []string{ + herculesBinary, + "-l", lAddr.String(), + "-i", iface, + } + + if herculesConfig != nil { + args = append(args, "-c", *herculesConfig) + } + if offset != -1 { + args = append(args, "-foffset", strconv.FormatInt(offset, 10)) + } + return args, nil +} + +// PrepareHerculesSendCommand builds an exec.Command to run Hercules in sender mode +// Does not attempt to resolve a configuration file, if herculesConfig is nil +func PrepareHerculesSendCommand(herculesBinary string, herculesConfig *string, localAddress *net.UDPAddr, remoteAddress *snet.UDPAddr, file string, offset int64) (*exec.Cmd, error) { + args, err := prepareHerculesArgs(herculesBinary, herculesConfig, localAddress, offset) + if err != nil { + return nil, err + } + + args = append(args, + "-t", file, + "-d", remoteAddress.String(), + ) + return exec.Command("sudo", args...), nil +} + +// PrepareHerculesRecvCommand builds an exec.Command to run Hercules in receiver mode +// Does not attempt to resolve a configuration file, if herculesConfig is nil +func PrepareHerculesRecvCommand(herculesBinary string, herculesConfig *string, localAddress *net.UDPAddr, file string, offset int64) (*exec.Cmd, error) { + args, err := prepareHerculesArgs(herculesBinary, herculesConfig, localAddress, offset) + if err != nil { + return nil, err + } + + args = append(args, + "-o", file, + "-timeout", "5", + ) + return exec.Command("sudo", args...), nil +} + +func checkIfRegularFile(fileName string) (bool, error) { + stat, err := os.Stat(fileName) + if err == nil && stat.Mode().IsRegular() { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +// ResolveConfig checks for a hercules.toml config file in the following locations: +// - the current working directory +// - /etc/scion-ftp/ +// - /etc/ +func ResolveConfig() (*string, error) { + candidates := []string{"hercules.toml", "/etc/hercules.toml", "/etc/scion-ftp/hercules.toml"} + for _, candidate := range candidates { + exists, err := checkIfRegularFile(candidate) + if err != nil { + return nil, err + } + if exists { + return &candidate, nil + } + } + return nil, nil +} + +// AssertFileWriteable checks that the file is writeable with the process owner's user permissions +// If the file does not exist, AssertFileWriteable will attempt to create it +func AssertFileWriteable(path string) (fileCreated bool, err error) { + fileCreated = false + f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0666) + if err != nil && errors.Is(err, os.ErrNotExist) { + f, err = os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) + if err != nil { + return + } + fileCreated = true + } + _ = f.Close() + return +} + +func OwnFile(file string) error { + usr, err := user.Current() + if err != nil { + return err + } + + args := []string{ + "chown", + fmt.Sprintf("%s:%s", usr.Uid, usr.Gid), + file, + } + + cmd := exec.Command("sudo", args...) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + return cmd.Run() +} diff --git a/internal/ftp/mode/mode.go b/internal/ftp/mode/mode.go new file mode 100644 index 00000000..cfd251a0 --- /dev/null +++ b/internal/ftp/mode/mode.go @@ -0,0 +1,23 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mode + +// FTP modes +const ( + Stream = 'S' + ExtendedBlockMode = 'E' + PartialFileTransport = "PFT" + Hercules = 'H' +) diff --git a/internal/ftp/socket/delayedclosersocket.go b/internal/ftp/socket/delayedclosersocket.go new file mode 100644 index 00000000..8f0bb393 --- /dev/null +++ b/internal/ftp/socket/delayedclosersocket.go @@ -0,0 +1,36 @@ +// Copyright 2021 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package socket + +import ( + "io" + "net" + "time" +) + +// DelayedCloserSocket is used to postpone calling Close() on an underlying IO that provides buffers we can't immediately free up. +type DelayedCloserSocket struct { + net.Conn + io.Closer + time.Duration +} + +func (s DelayedCloserSocket) Close() error { + go func() { + time.Sleep(s.Duration) + _ = s.Closer.Close() + }() + return s.Conn.Close() +} diff --git a/internal/ftp/socket/listener.go b/internal/ftp/socket/listener.go new file mode 100644 index 00000000..a694d841 --- /dev/null +++ b/internal/ftp/socket/listener.go @@ -0,0 +1,72 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package socket + +import ( + "context" + "crypto/tls" + "fmt" + + "github.com/lucas-clemente/quic-go" + + "github.com/netsec-ethz/scion-apps/pkg/appnet/appquic" +) + +type Listener struct { + QuicListener quic.Listener +} + +func ListenPort(port uint16, cert *tls.Certificate) (*Listener, error) { + tlsConfig := &tls.Config{ + NextProtos: []string{"scionftp"}, + Certificates: []tls.Certificate{*cert}, + } + + listener, err := appquic.ListenPort(port, tlsConfig, nil) + if err != nil { + return nil, fmt.Errorf("unable to listen: %s", err) + } + + return &Listener{ + listener, + }, nil +} + +func (listener *Listener) Close() error { + return listener.QuicListener.Close() +} + +// Accept accepts a QUIC session with exactly one stream on listener. +func (listener *Listener) Accept() (*SingleStream, error) { + session, err := listener.QuicListener.Accept(context.Background()) + if err != nil { + return nil, err + } + + stream, err := session.AcceptStream(context.Background()) + if err != nil { + return nil, err + } + + // AcceptStream() blocks until first data arrives, so we need to: + err = consumeHandshake(stream) + if err != nil { + return nil, err + } + return &SingleStream{ + Stream: stream, + session: session, + }, nil +} diff --git a/internal/ftp/socket/socket.go b/internal/ftp/socket/socket.go new file mode 100644 index 00000000..c20d8f42 --- /dev/null +++ b/internal/ftp/socket/socket.go @@ -0,0 +1,79 @@ +// Copyright 2018 The goftp Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. +// +// Copyright 2020 ETH Zurich modifications to add support for SCION +package socket + +import ( + "crypto/tls" + "errors" + "fmt" + "io" + "net" + + "github.com/lucas-clemente/quic-go" + + "github.com/netsec-ethz/scion-apps/pkg/appnet/appquic" +) + +var _ net.Conn = &SingleStream{} + +type SingleStream struct { + quic.Stream + session quic.Session +} + +func (s SingleStream) LocalAddr() net.Addr { + return s.session.LocalAddr() +} + +func (s SingleStream) RemoteAddr() net.Addr { + return s.session.RemoteAddr() +} + +func DialAddr(remoteAddr string) (*SingleStream, error) { + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + NextProtos: []string{"scionftp"}, + } + + quicConfig := &quic.Config{ + KeepAlive: true, + } + + session, err := appquic.Dial(remoteAddr, tlsConfig, quicConfig) + if err != nil { + return nil, fmt.Errorf("unable to dial %s: %s", remoteAddr, err) + } + + stream, err := session.OpenStream() + if err != nil { + return nil, err + } + err = sendHandshake(stream) // needed to unblock AcceptStream() + if err != nil { + return nil, err + } + + return &SingleStream{stream, session}, nil +} + +func sendHandshake(rw io.ReadWriter) error { + msg := []byte{200} + _, err := rw.Write(msg) + return err +} + +func consumeHandshake(rw io.ReadWriter) error { + msg := make([]byte, 1) + n, err := rw.Read(msg) + if err != nil { + return err + } + if n != 1 { + return errors.New("invalid handshake received") + } + + return nil +} diff --git a/internal/ftp/striping/header.go b/internal/ftp/striping/header.go new file mode 100644 index 00000000..ecaad7bf --- /dev/null +++ b/internal/ftp/striping/header.go @@ -0,0 +1,63 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package striping + +// Extended Block Header Flags +const ( + // BlockFlagEndOfDataCount uint8 = 64 + // BlockFlagSuspectErrors uint8 = 32 + BlockFlagEndOfData uint8 = 8 + // BlockFlagSenderClosesConnection uint8 = 4 + + // Deprecated: Around for legacy purposes + // BlockFlagEndOfRecord uint8 = 128 + // Deprecated: Around for legacy purposes + // BlockFlagRestartMarker uint8 = 16 +) + +// The header is sent over the data channels and indicates +// information about the following data (if any) +// See https://www.ogf.org/documents/GWD-R/GFD-R.020.pdf +// section "Extended Block Mode" +type Header struct { + Descriptor uint8 + ByteCount uint64 + OffsetCount uint64 +} + +func NewHeader(byteCount, offsetCount uint64, flags ...uint8) *Header { + header := Header{ + ByteCount: byteCount, + OffsetCount: offsetCount, + } + + header.AddFlag(flags...) + + return &header +} + +func (header *Header) ContainsFlag(flag uint8) bool { + return header.Descriptor&flag == flag +} + +func (header *Header) AddFlag(flags ...uint8) { + for _, flag := range flags { + header.Descriptor |= flag + } +} + +func (header *Header) GetEODCount() int { + return int(header.OffsetCount) +} diff --git a/internal/ftp/striping/heap.go b/internal/ftp/striping/heap.go new file mode 100644 index 00000000..9cf6b3b3 --- /dev/null +++ b/internal/ftp/striping/heap.go @@ -0,0 +1,40 @@ +package striping + +import ( + "container/heap" +) + +var _ heap.Interface = &segmentHeap{} + +type segmentHeap struct { + segments []Segment +} + +func (sh *segmentHeap) Len() int { + return len(sh.segments) +} + +func (sh *segmentHeap) Less(i, j int) bool { + return sh.segments[i].OffsetCount < sh.segments[j].OffsetCount +} + +func (sh *segmentHeap) Swap(i, j int) { + sh.segments[i], sh.segments[j] = sh.segments[j], sh.segments[i] +} + +func (sh *segmentHeap) Push(x interface{}) { + sh.segments = append(sh.segments, x.(Segment)) +} + +func (sh *segmentHeap) Pop() interface{} { + s := sh.segments[sh.Len()-1] + sh.segments = sh.segments[:sh.Len()-1] + return s +} + +func newSegmentHeap() *segmentHeap { + a := make([]Segment, 0) + return &segmentHeap{ + a, + } +} diff --git a/internal/ftp/striping/multisocket.go b/internal/ftp/striping/multisocket.go new file mode 100644 index 00000000..58baaffe --- /dev/null +++ b/internal/ftp/striping/multisocket.go @@ -0,0 +1,75 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package striping + +import ( + "net" + "time" +) + +type MultiSocket struct { + *readerSocket + *writerSocket +} + +func (m *MultiSocket) SetReadDeadline(t time.Time) error { + // readerSocket.sockets contains the same sockets as writerSocket.sockets, hence it's fine to just: + for _, s := range m.readerSocket.sockets { + if err := s.SetReadDeadline(t); err != nil { + return err + } + } + return nil +} + +func (m *MultiSocket) SetWriteDeadline(t time.Time) error { + // writerSocket.sockets contains the same sockets as readerSocket.sockets, hence it's fine to just: + for _, s := range m.writerSocket.sockets { + if err := s.SetWriteDeadline(t); err != nil { + return err + } + } + return nil +} + +func (m *MultiSocket) SetDeadline(t time.Time) error { + if err := m.SetReadDeadline(t); err != nil { + return err + } + return m.SetWriteDeadline(t) +} + +var _ net.Conn = &MultiSocket{} + +// Only the client should close the socket +// Sends the closing message +func (m *MultiSocket) Close() error { + return m.writerSocket.Close() +} + +func (m *MultiSocket) LocalAddr() net.Addr { + return m.writerSocket.sockets[0].LocalAddr() +} + +func (m *MultiSocket) RemoteAddr() net.Addr { + return m.writerSocket.sockets[0].RemoteAddr() +} + +func NewMultiSocket(sockets []net.Conn, maxLength int) *MultiSocket { + return &MultiSocket{ + newReaderSocket(sockets), + newWriterSocket(sockets, maxLength), + } +} diff --git a/internal/ftp/striping/queue.go b/internal/ftp/striping/queue.go new file mode 100644 index 00000000..0172705e --- /dev/null +++ b/internal/ftp/striping/queue.go @@ -0,0 +1,90 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package striping + +import ( + "container/heap" +) + +type SegmentQueue struct { + internal *segmentHeap + offset uint64 + openStreams int +} + +func (q *SegmentQueue) Push(segment Segment) { + heap.Push(q.internal, segment) +} + +func (q *SegmentQueue) Pop() Segment { + return heap.Pop(q.internal).(Segment) +} + +func (q *SegmentQueue) Peek() Segment { + return q.internal.segments[0] +} + +func (q *SegmentQueue) Len() int { + return q.internal.Len() +} + +func NewSegmentQueue(workers int) *SegmentQueue { + return &SegmentQueue{ + internal: newSegmentHeap(), + offset: 0, + openStreams: workers, + } +} + +func (q *SegmentQueue) PushChannel() (chan<- Segment, <-chan Segment) { + // Make buffered channels 4 times as large as the number of streams + push := make(chan Segment, q.openStreams*4) + pop := make(chan Segment, q.openStreams*4) + go func() { + for { + // Has received everything + if q.openStreams == 0 && q.Len() == 0 { + close(push) + close(pop) + return + } + + // Empty packet + if q.Len() > 0 && q.Peek().ByteCount == 0 { + q.Pop() + } else if q.Len() > 0 && q.offset == q.Peek().OffsetCount { + select { + // Do not want to op if case not selected + case pop <- q.Peek(): + sent := q.Pop() + q.offset += sent.ByteCount + case next := <-push: + q.handleSegment(next) + } + } else if q.openStreams > 0 { + q.handleSegment(<-push) + } + } + }() + + return push, pop +} + +func (q *SegmentQueue) handleSegment(next Segment) { + if next.ContainsFlag(BlockFlagEndOfData) { + q.openStreams-- + } + q.Push(next) +} diff --git a/internal/ftp/striping/readsocket.go b/internal/ftp/striping/readsocket.go new file mode 100644 index 00000000..236a3426 --- /dev/null +++ b/internal/ftp/striping/readsocket.go @@ -0,0 +1,64 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package striping + +import ( + "io" + "net" +) + +type readerSocket struct { + sockets []net.Conn + queue *SegmentQueue + pop <-chan Segment +} + +var _ io.Reader = &readerSocket{} +var _ io.Closer = &readerSocket{} + +func newReaderSocket(sockets []net.Conn) *readerSocket { + return &readerSocket{ + sockets: sockets, + queue: NewSegmentQueue(len(sockets)), + } +} + +func (s *readerSocket) Read(p []byte) (n int, err error) { + + if s.pop == nil { + push, pop := s.queue.PushChannel() + s.pop = pop + + for _, subSocket := range s.sockets { + reader := newReadWorker(subSocket) + go reader.Run(push) + } + } + + next := <-s.pop + + // Channel has been closed -> no more segments + if next.Header == nil { + return 0, io.EOF + } + + // If copy copies less then the ByteCount we have a problem + return copy(p, next.Data), nil + +} + +func (s *readerSocket) Close() error { + panic("implement me") +} diff --git a/internal/ftp/striping/readworker.go b/internal/ftp/striping/readworker.go new file mode 100644 index 00000000..415b7d91 --- /dev/null +++ b/internal/ftp/striping/readworker.go @@ -0,0 +1,78 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package striping + +import ( + "encoding/binary" + "fmt" + "net" +) + +// A readWorker should be dispatched and runs until it +// receives the closing connection +// Does not need to be closed since it's closed +// automatically +type readWorker struct { + socket net.Conn + // ctx context.Context // Currently unused +} + +func newReadWorker(socket net.Conn) *readWorker { + return &readWorker{socket: socket} +} + +// Keeps running until it receives an EOD flag +func (s *readWorker) Run(push chan<- Segment) { + for { + seg, err := receiveNextSegment(s.socket) + if err != nil { + fmt.Printf("Failed to receive segment: %s\n", err) + return + } + + push <- seg + + if seg.ContainsFlag(BlockFlagEndOfData) { + return + } + + } +} + +func receiveNextSegment(socket net.Conn) (Segment, error) { + header := &Header{} + err := binary.Read(socket, binary.BigEndian, header) + if err != nil { + return Segment{}, fmt.Errorf("failed to read header: %s", err) + } + + data := make([]byte, header.ByteCount) + cur := 0 + + // Read all bytes + for { + if cur < int(header.ByteCount) { + n, err := socket.Read(data[cur:header.ByteCount]) + if err != nil { + return Segment{}, fmt.Errorf("failed to read payload: %s", err) + } + + cur += n + } + if cur == int(header.ByteCount) { + return NewSegmentWithHeader(header, data), nil + } + } +} diff --git a/internal/ftp/striping/segment.go b/internal/ftp/striping/segment.go new file mode 100644 index 00000000..41427387 --- /dev/null +++ b/internal/ftp/striping/segment.go @@ -0,0 +1,36 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package striping + +type Segment struct { + *Header + Data []byte +} + +func NewSegment(data []byte, offset int, flags ...uint8) Segment { + + return Segment{ + NewHeader(uint64(len(data)), uint64(offset), flags...), + data, + } + +} + +func NewSegmentWithHeader(header *Header, data []byte) Segment { + return Segment{ + header, + data, + } +} diff --git a/internal/ftp/striping/writesocket.go b/internal/ftp/striping/writesocket.go new file mode 100644 index 00000000..5020dd04 --- /dev/null +++ b/internal/ftp/striping/writesocket.go @@ -0,0 +1,117 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package striping + +import ( + "io" + "net" + "sync" +) + +type writerSocket struct { + sockets []net.Conn + maxLength int + segmentChannel chan Segment + wg *sync.WaitGroup + written int + dispatchedWriters bool +} + +var _ io.Writer = &writerSocket{} +var _ io.Closer = &writerSocket{} + +func newWriterSocket(sockets []net.Conn, maxLength int) *writerSocket { + return &writerSocket{ + sockets: sockets, + maxLength: maxLength, + segmentChannel: make(chan Segment), + wg: &sync.WaitGroup{}, + } +} + +// Will dispatch workers if required and write on +// the allocated stream. After writing it is necessary +// to call FinishAndWait() to make sure that everything is sent +func (s *writerSocket) Write(p []byte) (n int, err error) { + if !s.dispatchedWriters { + s.dispatchedWriters = true + s.dispatchWriter() + } + + cur := 0 + + for { + if cur == len(p) { + return cur, nil + } + + to := cur + s.maxLength + if to > len(p) { + to = len(p) + } + + data := make([]byte, to-cur) + copy(data, p[cur:to]) + + s.segmentChannel <- NewSegment(data, s.written) + + s.written += to - cur + + cur = to + } +} + +func (s *writerSocket) FinishAndWait() { + // Wait until all writers have finished + if !s.dispatchedWriters { + return + } + + close(s.segmentChannel) + s.wg.Wait() + + s.dispatchedWriters = false +} + +func (s *writerSocket) dispatchWriter() { + for _, socket := range s.sockets { + s.wg.Add(1) + worker := newWriteWorker(s.wg, s.segmentChannel, socket) + go worker.Run() + } +} + +// Closing the writerSocket blocks until until all +// children have finished sending and then closes +// all sub-sockets +func (s *writerSocket) Close() error { + + s.FinishAndWait() + + var errs []error + + for i := range s.sockets { + err := s.sockets[i].Close() + if err != nil { + errs = append(errs, err) + } + } + + if len(errs) == 0 { + return nil + } else { + return errs[0] + } +} diff --git a/internal/ftp/striping/writeworker.go b/internal/ftp/striping/writeworker.go new file mode 100644 index 00000000..52710aff --- /dev/null +++ b/internal/ftp/striping/writeworker.go @@ -0,0 +1,87 @@ +// Copyright 2020 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package striping + +import ( + "encoding/binary" + "net" + "sync" + + "github.com/scionproto/scion/go/lib/log" +) + +type writeWorker struct { + wg *sync.WaitGroup + segments chan Segment + socket net.Conn +} + +func newWriteWorker(wg *sync.WaitGroup, segments chan Segment, socket net.Conn) *writeWorker { + return &writeWorker{wg, segments, socket} +} + +// Writes segments until receives cancellation signal on Done() +// and sends EOD Header after that. +func (w *writeWorker) Run() { + for { + segment, more := <-w.segments + if segment.Header != nil { + err := w.writeSegment(segment) + if err != nil { + log.Error("Failed to write segment", "err", err) + } + } + + if !more { + eod := NewHeader(0, 0, BlockFlagEndOfData) + err := w.writeHeader(eod) + if err != nil { + log.Error("Failed to write eod header", "err", err) + } + w.wg.Done() + return + } + } +} + +func (w *writeWorker) writeHeader(header *Header) error { + return binary.Write(w.socket, binary.BigEndian, header) +} + +func (w *writeWorker) writeSegment(segment Segment) error { + err := w.writeHeader(segment.Header) + if err != nil { + return err + } + + cur := 0 + + for { + + n, err := w.socket.Write(segment.Data[cur:segment.ByteCount]) + if err != nil { + return err + } + + cur += n + + if cur == int(segment.ByteCount) { + break + } + + } + + return nil +} diff --git a/pkg/appnet/appquic/appquic.go b/pkg/appnet/appquic/appquic.go index 11df43a9..e64eb495 100644 --- a/pkg/appnet/appquic/appquic.go +++ b/pkg/appnet/appquic/appquic.go @@ -20,6 +20,7 @@ package appquic import ( "crypto/tls" "fmt" + "net" "sync" "github.com/lucas-clemente/quic-go" @@ -127,6 +128,17 @@ func ensurePathDefined(raddr *snet.UDPAddr) error { return nil } +// Listen listens for QUIC connections on a SCION/UDP address. +// +// See note on wildcard addresses in the appnet package documentation. +func Listen(listen *net.UDPAddr, tlsConf *tls.Config, quicConfig *quic.Config) (quic.Listener, error) { + sconn, err := appnet.Listen(listen) + if err != nil { + return nil, err + } + return quic.Listen(sconn, tlsConf, quicConfig) +} + // ListenPort listens for QUIC connections on a SCION/UDP port. // // See note on wildcard addresses in the appnet package documentation.