From 921dcec5ea76f63cf19fb023189f637b7c2363ce Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 8 Feb 2017 17:56:05 +0100 Subject: [PATCH] Working CWL files Refs #4 --- Dockerfile | 8 +-- README.md | 6 ++ .../esciencecenter/xenon/cli/CopyCommand.java | 2 +- .../xenon/cli/DownloadCommand.java | 2 +- .../nl/esciencecenter/xenon/cli/Main.java | 2 +- .../xenon/cli/UploadOutput.java | 4 +- xenon-download.cwl | 69 ++++++++++++++---- xenon-ls.cwl | 61 +++++++++++++--- xenon-upload.cwl | 71 +++++++++++++++---- 9 files changed, 175 insertions(+), 50 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5254c38..08be312 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,6 @@ # Using `./gradlew docker` to build image FROM openjdk:jre-alpine MAINTAINER Stefan Verhoeven -COPY bin /app/bin -COPY lib /app/lib -WORKDIR /app -ENTRYPOINT ["/app/bin/xenon"] -CMD ["--help"] +COPY bin /usr/bin +COPY lib /usr/lib +CMD ["/usr/bin/xenon", "--help"] diff --git a/README.md b/README.md index 2b95acd..fbd8d75 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,12 @@ xenon --adaptor ssh --json list --location localhost /etc xenon --adaptor ssh --json upload --source /etc/passwd --location localhost --path /tmp/copy-of-passwd ``` +To keep password or passphrase invisible in process list put the password in a text file (eg. 'password.txt') and then use '@password.txt' as argument. +For example: +``` +xenon --username $USER --password @password.txt sftp --location localhost list $PWD/src +``` + # Build ``` diff --git a/src/main/java/nl/esciencecenter/xenon/cli/CopyCommand.java b/src/main/java/nl/esciencecenter/xenon/cli/CopyCommand.java index 13de6e3..8847d81 100644 --- a/src/main/java/nl/esciencecenter/xenon/cli/CopyCommand.java +++ b/src/main/java/nl/esciencecenter/xenon/cli/CopyCommand.java @@ -43,7 +43,7 @@ protected void copy(Files files, CopyInput source, CopyInput target, Boolean rec if (!target.path.startsWith("/") && !target.path.equals("-")) { // Path is relative to working directory, make it absolute RelativePath workingDirectory = new RelativePath(System.getProperty("user.dir")); - sourcePath = files.newPath(sourceFS, workingDirectory.resolve(target.path)); + targetPath = files.newPath(targetFS, workingDirectory.resolve(target.path)); } } diff --git a/src/main/java/nl/esciencecenter/xenon/cli/DownloadCommand.java b/src/main/java/nl/esciencecenter/xenon/cli/DownloadCommand.java index ffaf38a..0a6be1d 100644 --- a/src/main/java/nl/esciencecenter/xenon/cli/DownloadCommand.java +++ b/src/main/java/nl/esciencecenter/xenon/cli/DownloadCommand.java @@ -23,7 +23,7 @@ public Subparser buildArgumentParser(Subparsers subparsers) { .description("Download source file to local file"); subparser.addArgument("source").help("Local source path").required(true); subparser.addArgument("target") - .type(Arguments.fileType().acceptSystemIn().verifyCanWriteParent()) + .type(Arguments.fileType().acceptSystemIn()) .help("Target path, use '-' for stdout") .required(true); subparser.addArgument("--recursive").help("Copy directories recursively").action(Arguments.storeTrue()); diff --git a/src/main/java/nl/esciencecenter/xenon/cli/Main.java b/src/main/java/nl/esciencecenter/xenon/cli/Main.java index c82d499..c3c1397 100644 --- a/src/main/java/nl/esciencecenter/xenon/cli/Main.java +++ b/src/main/java/nl/esciencecenter/xenon/cli/Main.java @@ -32,7 +32,7 @@ public static void main(String[] args) throws Exception { Main main = new Main(); ArgumentParser parser = main.buildArgumentParser(); Namespace res = parser.parseArgsOrFail(args); - LOGGER.warn(res.toString()); + LOGGER.debug(res.toString()); ICommand subcommand = res.get("command"); Xenon xenon = XenonFactory.newXenon(buildXenonProperties(res)); subcommand.run(res, xenon); diff --git a/src/main/java/nl/esciencecenter/xenon/cli/UploadOutput.java b/src/main/java/nl/esciencecenter/xenon/cli/UploadOutput.java index 1fac154..b6165df 100644 --- a/src/main/java/nl/esciencecenter/xenon/cli/UploadOutput.java +++ b/src/main/java/nl/esciencecenter/xenon/cli/UploadOutput.java @@ -1,12 +1,12 @@ package nl.esciencecenter.xenon.cli; public class UploadOutput { - public CwlFile source; + public String source; public String location; public String path; public UploadOutput(CopyInput source, CopyInput target) { - this.source = new CwlFile(source.path); + this.source = source.path; location = target.location; path = target.path; } diff --git a/xenon-download.cwl b/xenon-download.cwl index 32cc614..d57d6c5 100755 --- a/xenon-download.cwl +++ b/xenon-download.cwl @@ -1,40 +1,81 @@ #!/usr/bin/env cwl-runner +# Native: +# xenon --format cwljson sftp --location localhost download $PWD/README.md copy-of-README.md +# Using cwl-runnner: +# ./xenon-download.cwl --certfile ~/.ssh/id_rsa --scheme sftp --location $USER@172.17.0.1 --source $PWD/README.md --target copy-of-README.md cwlVersion: v1.0 class: CommandLineTool doc: Download file from remote storage -requirements: +hints: - class: DockerRequirement # xenon-cli Docker container needs to be manually build before dockerImageId: nlesc/xenon-cli +baseCommand: xenon arguments: -- --json -- download +- prefix: --format + valueFrom: cwljson + position: 0 +- valueFrom: download + position: 3 inputs: - scheme: - type: string - inputBinding: - prefix: --scheme - position: -1 certfile: type: File? + doc: Certificate file inputBinding: prefix: --certfile - position: -1 - location: + position: 0 + username: type: string? inputBinding: - position: 1 - path: + prefix: --username + position: 0 + password: + type: string? + doc: Password, watch out do not use on systems with untrusted users + inputBinding: + prefix: --password + position: 0 + scheme: type: string + doc: Scheme, eg. file, sftp, ftp inputBinding: + position: 1 + location: + type: string? + doc: List contents of path at location + inputBinding: + prefix: --location position: 2 +# TODO prop should be optional, atm is must be set +# prop: +# doc: Xenon adaptor properties +# type: +# type: array +# items: string +# inputBinding: +# prefix: --prop +# inputBinding: +# position: 2 + overwrite: + type: boolean + inputBinding: + prefix: --overwrite + position: 4 + ignore: + type: boolean + inputBinding: + prefix: --ignore + position: 4 + source: + type: string + inputBinding: + position: 5 target: type: string inputBinding: - position: 3 + position: 6 outputs: target: type: File outputBinding: glob: $(inputs.target) -# stdout: cwl.output.json diff --git a/xenon-ls.cwl b/xenon-ls.cwl index 7166ddd..0cc6cff 100755 --- a/xenon-ls.cwl +++ b/xenon-ls.cwl @@ -1,33 +1,72 @@ #!/usr/bin/env cwl-runner +# Native: +# xenon --format cwljson sftp --location localhost list $PWD/ +# Using cwl-runnner: +# ./xenon-ls.cwl --scheme sftp --location $USER@172.17.0.1 --path $PWD --certfile ~/.ssh/id_rsa cwlVersion: v1.0 class: CommandLineTool doc: List objects on remote storage -requirements: +hints: - class: DockerRequirement # xenon-cli Docker container needs to be manually build before dockerImageId: nlesc/xenon-cli +baseCommand: xenon arguments: -- --json -- list +- prefix: --format + valueFrom: cwljson + position: 0 +- valueFrom: list + position: 3 inputs: - scheme: - type: string - inputBinding: - prefix: --scheme - position: -1 certfile: type: File? + doc: Certificate file inputBinding: prefix: --certfile - position: -1 - location: + position: 0 + username: type: string? + inputBinding: + prefix: --username + position: 0 + password: + type: string? + doc: Password, watch out do not use on systems with untrusted users + inputBinding: + prefix: --password + position: 0 + scheme: + type: string + doc: Scheme, eg. file, sftp, ftp inputBinding: position: 1 + location: + type: string? + doc: List contents of path at location + inputBinding: + prefix: --location + position: 2 +# TODO prop should be optional, atm is must be set +# prop: +# doc: Xenon adaptor properties +# type: +# type: array +# items: string +# inputBinding: +# prefix: --prop +# inputBinding: +# position: 2 + recursive: + doc: List directories recursively + type: boolean? + inputBinding: + prefix: --recursive + position: 4 path: type: string + doc: List contents of path at location inputBinding: - position: 2 + position: 5 outputs: objects: type: diff --git a/xenon-upload.cwl b/xenon-upload.cwl index f7147c3..3738d83 100755 --- a/xenon-upload.cwl +++ b/xenon-upload.cwl @@ -1,37 +1,78 @@ #!/usr/bin/env cwl-runner +# Native: +# xenon --format cwljson sftp --location localhost upload README.md $PWD/copy-of-README.md +# Using cwl-runnner: +# ./xenon-upload.cwl --certfile ~/.ssh/id_rsa --scheme sftp --location $USER@172.17.0.1 --source README.md --target $PWD/copy-of-README.md cwlVersion: v1.0 class: CommandLineTool doc: Upload file to remote storage -requirements: +hints: - class: DockerRequirement # xenon-cli Docker container needs to be manually build before dockerImageId: nlesc/xenon-cli +baseCommand: xenon arguments: -- --json -- upload +- prefix: --format + valueFrom: cwljson + position: 0 +- valueFrom: upload + position: 3 inputs: - scheme: - type: string - inputBinding: - prefix: --scheme - position: -1 certfile: type: File? + doc: Certificate file inputBinding: prefix: --certfile - position: -1 - source: - type: File + position: 0 + username: + type: string? + inputBinding: + prefix: --username + position: 0 + password: + type: string? + doc: Password, watch out do not use on systems with untrusted users + inputBinding: + prefix: --password + position: 0 + scheme: + type: string + doc: Scheme, eg. file, sftp, ftp inputBinding: position: 1 location: type: string? + doc: List contents of path at location inputBinding: + prefix: --location position: 2 - path: +# TODO prop should be optional, atm is must be set +# prop: +# doc: Xenon adaptor properties +# type: +# type: array +# items: string +# inputBinding: +# prefix: --prop +# inputBinding: +# position: 2 + overwrite: + type: boolean + inputBinding: + prefix: --overwrite + position: 4 + ignore: + type: boolean + inputBinding: + prefix: --ignore + position: 4 + source: + type: File + inputBinding: + position: 5 + target: type: string inputBinding: - position: 3 -outputs: - path: File + position: 6 +outputs: [] stdout: cwl.output.json