diff --git a/CHANGELOG b/CHANGELOG index d860259..7bff80e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,2 +1,10 @@ +* Wed Mar 11 2020 Matiss Treinis - 1.1.0 +- Logging timer no longer reports nonsense values (#1). +- Environment name is now available from within scripts (#2). +- Archives created using the API now have proper timestamps and metadata (#3). +- Empty error messages are no longer possible in libkafe to avoid ambiguous errors (#4). +- Extra arguments to "do" command are now forwarded to task function (#5). +- SSH key passphrase and user password can now be defined in executing environment (#6). + * Wed Mar 4 2020 Matiss Treinis - 1.0.0 - Initial public release diff --git a/CMakeLists.txt b/CMakeLists.txt index efc5901..57867c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.11.4) # IMPORTANT: updating version might require update in package dependencies at the end of this file. -set(KAFE_VERSION "1.0.0") +set(KAFE_VERSION "1.1.0") set(KAFE_VERSION_INT 10) set(KAFE_VERSION_DEP_NEXT_MAJOR "2.0.0") diff --git a/DOWNLOAD.md b/DOWNLOAD.md new file mode 100644 index 0000000..f032e9d --- /dev/null +++ b/DOWNLOAD.md @@ -0,0 +1,84 @@ +### Latest stable release downloads +#### Latest stable build is v1.1.0 + +Here you can find links to latest binary packages for all supported operating systems. + +- [CentOS and RHEL 7](#centos-and-rhel-7) +- [CentOS and RHEL 8](#centos-and-rhel-8) +- [Debian 9](#debian-9) +- [Debian 10](#debian-10) +- [Debian 11](#debian-11) +- [Fedora 31](#fedora-31) +- [Fedora 32](#fedora-32) +- [Fedora 33](#fedora-33) +- [Ubuntu 18.04](#ubuntu-1804) +- [Ubuntu 19.10](#ubuntu-1910) +- [Ubuntu 20.04](#ubuntu-2004) + +*NOTE:* macOS is supported, but there are no binary builds available. See [README](./README.md#building-on-macos) +on how to build macOS binaries from source. + +*NOTE:* You should be able to use these binary packages for any derivative distributions too. For example, +Elementary OS 5.1 users can use Ubuntu 18.04 packages, since Elementary OS 5.1 is based on Ubuntu 18.04. + +See [releases](https://github.com/libkafe/kafe/releases) for all available downloads and historic versions, +including development headers. + +#### CentOS and RHEL 7 + +- [kafe-cli-1.1.0-1.x86_64.el7.rpm](https://github.com/libkafe/kafe/releases/download/v1.1.0/kafe-cli-1.1.0-1.x86_64.el7.rpm) +- [libkafe-1.1.0-1.x86_64.el7.rpm](https://github.com/libkafe/kafe/releases/download/v1.1.0/libkafe-1.1.0-1.x86_64.el7.rpm) + +**IMPORTANT:** Kafe requires Lua version 5.3, a dependency not available in EL7 by default. You can either +build the package yourself or use [Cheese](http://www.nosuchhost.net/~cheese/fedora/packages/epel-7/x86_64/cheese-release.html) +repository to obtain compatible Lua package. + +#### CentOS and RHEL 8 + +- [kafe-cli-1.1.0-1.x86_64.el8.rpm](https://github.com/libkafe/kafe/releases/download/v1.1.0/kafe-cli-1.1.0-1.x86_64.el8.rpm) +- [libkafe-1.1.0-1.x86_64.el8.rpm](https://github.com/libkafe/kafe/releases/download/v1.1.0/libkafe-1.1.0-1.x86_64.el8.rpm) + +#### Debian 9 + +- [kafe-cli_1.1.0_amd64.deb9.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/kafe-cli_1.1.0_amd64.deb9.deb) +- [libkafe_1.1.0_amd64.deb9.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/libkafe_1.1.0_amd64.deb9.deb) + +#### Debian 10 + +- [kafe-cli_1.1.0_amd64.deb10.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/kafe-cli_1.1.0_amd64.deb10.deb) +- [libkafe_1.1.0_amd64.deb10.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/libkafe_1.1.0_amd64.deb10.deb) + +#### Debian 11 + +- [kafe-cli_1.1.0_amd64.deb11.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/kafe-cli_1.1.0_amd64.deb11.deb) +- [libkafe_1.1.0_amd64.deb11.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/libkafe_1.1.0_amd64.deb11.deb) + +#### Fedora 31 + +- [kafe-cli-1.1.0-1.x86_64.f31.rpm](https://github.com/libkafe/kafe/releases/download/v1.1.0/kafe-cli-1.1.0-1.x86_64.f31.rpm) +- [libkafe-1.1.0-1.x86_64.f31.rpm](https://github.com/libkafe/kafe/releases/download/v1.1.0/libkafe-1.1.0-1.x86_64.f31.rpm) + +#### Fedora 32 + +- [kafe-cli-1.1.0-1.x86_64.f32.rpm](https://github.com/libkafe/kafe/releases/download/v1.1.0/kafe-cli-1.1.0-1.x86_64.f32.rpm) +- [libkafe-1.1.0-1.x86_64.f32.rpm](https://github.com/libkafe/kafe/releases/download/v1.1.0/libkafe-1.1.0-1.x86_64.f32.rpm) + +#### Fedora 33 + +- [kafe-cli-1.1.0-1.x86_64.f33.rpm](https://github.com/libkafe/kafe/releases/download/v1.1.0/kafe-cli-1.1.0-1.x86_64.f33.rpm) +- [libkafe-1.1.0-1.x86_64.f33.rpm](https://github.com/libkafe/kafe/releases/download/v1.1.0/libkafe-1.1.0-1.x86_64.f33.rpm) + +#### Ubuntu 18.04 + +- [kafe-cli_1.1.0_amd64.ubu1804.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/kafe-cli_1.1.0_amd64.ubu1804.deb) +- [libkafe_1.1.0_amd64.ubu1804.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/libkafe_1.1.0_amd64.ubu1804.deb) + +#### Ubuntu 19.10 + +- [kafe-cli_1.1.0_amd64.ubu1910.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/kafe-cli_1.1.0_amd64.ubu1910.deb) +- [libkafe_1.1.0_amd64.ubu1910.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/libkafe_1.1.0_amd64.ubu1910.deb) + +#### Ubuntu 20.04 + +- [kafe-cli_1.1.0_amd64.ubu2004.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/kafe-cli_1.1.0_amd64.ubu2004.deb) +- [libkafe_1.1.0_amd64.ubu2004.deb](https://github.com/libkafe/kafe/releases/download/v1.1.0/libkafe_1.1.0_amd64.ubu2004.deb) diff --git a/README.md b/README.md index 53eb17e..92fc9fe 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,15 @@ remote systems administration tasks. ## Downloads -See [releases](https://github.com/libkafe/kafe/releases). +See [available binary packages](DOWNLOAD.md) for current stable release or [releases](https://github.com/libkafe/kafe/releases) for +all available downloads and historic versions. Binary builds are available for: -- **CentOS** and **RHEL** versions 71, 8 -- **Fedora** versions 31, 32, 33 -- **Ubuntu** versions 18.04, 19.10, 20.04 -- **Debian** versions 9, 10, 11 +- **CentOS** and **RHEL** versions [71](./DOWNLOAD.md#centos-and-rhel-7), [8](./DOWNLOAD.md#centos-and-rhel-8) +- **Fedora** versions [31](./DOWNLOAD.md#fedora-31), [32](./DOWNLOAD.md#fedora-32), [33](./DOWNLOAD.md#fedora-33) +- **Ubuntu** versions [18.04](./DOWNLOAD.md#ubuntu-1804), [19.10](./DOWNLOAD.md#ubuntu-1910), [20.04](./DOWNLOAD.md#ubuntu-2004) +- **Debian** versions [9](./DOWNLOAD.md#debian-9), [10](./DOWNLOAD.md#debian-10), [11](./DOWNLOAD.md#debian-11) You should be able to use these binary packages for any derivative distributions. For example, Elementary OS 5.1 users can use Ubuntu 18.04 packages, since Elementary OS is based on Ubuntu 18.04. @@ -93,6 +94,19 @@ execute following Kafe CLI command: When executed, Kafe CLI will look for a file named `kafe.lua` in the current working direcory. This file will be interpreted and requested tasks from it will be executed against all relevant remote servers. +#### SSH and SSH authentication + +Kafe is using SSH for remote command execution and file uploads. It will attempt to use SSH agent, any known local +SSH keys, as well as GSSAPI-WITH-MIC and password based authentication to authenticate to remote hosts. + +If no SSH agent is present, you can set the passphrase to be used to decrypt any encrypted private keys using +environment variable `KAFE_SSH_PKEY_PASS`. You can set password to be used for password based authentication using +environment variable named `KAFE_SSH_USER_PASS`. + +**IMPORTANT:** Kafe will not automatically add remote keys to known hosts nor will it provide a way to do so interactively. +It is your responsibility to ensure remote host keys are added to known hosts before attempting to connect to remote hosts +using Kafe. Any attempts to connect to remote hosts with unknown or changed host keys will fail. + ### Debugging You can change the logging level of the CLI tool by setting `KAFE_LOG_LEVEL` environment variable. For example: diff --git a/cli/logger.cpp b/cli/logger.cpp index bec0dbe..d71581d 100644 --- a/cli/logger.cpp +++ b/cli/logger.cpp @@ -78,14 +78,14 @@ namespace kafe { static string context_to_s(const vector &context) { if (context.empty()) { - return ""; + return {}; } ostringstream ss; ss << IO_TTY_ANSI_COLOR_MAGENTA; - for (auto &ctx : context) { + for (const auto &ctx : context) { ss << '[' << ctx << "] "; } @@ -108,7 +108,7 @@ namespace kafe { case ALL: case TRACE: case DEBUG: - color = ""; + color = {}; break; case INFO: color = IO_TTY_ANSI_COLOR_BLUE; @@ -188,7 +188,7 @@ namespace kafe { line.c_str() ); - if (len > 1 && line[len - 1] != '\n') { + if (len > 0 && line[len - 1] != '\n') { fputs("\n", stdout); } diff --git a/cli/main.cpp b/cli/main.cpp index b183690..df0d829 100644 --- a/cli/main.cpp +++ b/cli/main.cpp @@ -117,10 +117,11 @@ int main(int argc, char *argv[]) { } if (0 == strcmp("do", argv[1])) { - if (4 != argc) { - cerr << "Command expects exactly two arguments - environment name an" - " comma separated task list.\n" - "Example: kafe do staging task1,task2,task3"; + if (4 > argc) { + cerr << "Command expects exactly at least two arguments - environment name and " + "a comma separated task list with any number of arbitrary arguments " + "to forward to the tasks being invoked.\n" + "Example: kafe do staging task1,task2,task3 "; print_usage(); return 1; } @@ -136,19 +137,23 @@ int main(int argc, char *argv[]) { envvals.insert(p); } - string environment = argv[2]; string task_list_s = argv[3]; vector task_list_v = split_csv_arguments(task_list_s, ','); + vector extra_args; + for (int ii = 4; ii < argc; ++ii) { + extra_args.emplace_back(argv[ii]); + } + try { auto project = Project("kafe.lua"); auto logger = Logger(); auto context = Context(envvals, environment, task_list_v, &logger); auto inventory = Inventory(); - project.execute(context, inventory); + project.execute(context, inventory, extra_args); } catch (RuntimeException &e) { - cerr << e.what(); + cerr << e.what() << endl; return 1; } return 0; diff --git a/docs/SCRIPTING_API_L1.md b/docs/SCRIPTING_API_L1.md index 5e343f7..f28f182 100644 --- a/docs/SCRIPTING_API_L1.md +++ b/docs/SCRIPTING_API_L1.md @@ -35,7 +35,57 @@ symlink all the deployment on all servers, reload services on all servers. SSH connections to remote servers are reused - one remote server with unique combination of host, user and port will only have one connection created regardless of environment and role. -## Available API values +## Defining tasks + +Kafe tasks are defined in a file called `kafe.lua`. This file is usually placed in a directory that might or might not +contain additional content related to the script itself. For example when automating software project deployments +you could place `kafe.lua` at the root directory of your software project. + +Kafe CLI will always look for a file named `kafe.lua` in the current working directory. + +Tasks are defined at the root level of the file by invoking `k.task(...)` method. + +```lua +local k = require('kafe') + +k.task('hello_world_task', function() + -- Your code here. +end) +``` + +### Passing arguments to tasks from command line + +It is possible to pass arbitrary arguments to the tasks when invoking them from command line. +Arguments are forwarded verbatim without modification, and are always of type string. + +Consider a task defined like in this example: + +```lua +local k = require('kafe') + +k.task('hello_world_task', function(arg1, arg2) + print(arg1, arg2) + -- Your code here. +end) +``` + +When invoked by command line `kafe do staging hello_world_task hello world` this task will receive +arguments `arg1` with value `hello` and `arg2` with value `world`. + +**IMPORTANT:** task arguments are never mandatory. You have to validate arguments received by the task function +before using them. + +**IMPORTANT:** take care when invoking multiple tasks at the same time. All tasks requested will receive +all arguments as forwarded from the CLI. + +## Global values + +#### array[string] arg + +Program arguments where index `0` is full path to the current +script (`/kafe.lua`) with subsequent values (if any) containing extra arguments provided to the `do` command. + +## API values #### string k.version @@ -57,7 +107,11 @@ Release version of libkafe. Current API level. -## Available API methods +#### int k.environment + +Current environment name. + +## API methods ### void k.require_api(int version) @@ -67,6 +121,13 @@ Results in hard failure if: - API level of the environment is less than the one requested. +##### An example of usage + +```lua +local k = require('kafe') +k.require_api(1) -- Fails if API compatibility level is not 1 +``` + ### void k.task(string name, function callable) Define a task that can be invoked when the script is executed. @@ -75,6 +136,15 @@ Results in hard failure if: - Task with given name already exists. +##### An example of usage + +```lua +local k = require('kafe') +k.task('example_name', function() + -- Your code here +end) +``` + ### void k.add_inventory(string user, string host, int port, string env, string role) Add a server to inventory of given environment with given role. @@ -84,6 +154,16 @@ Results in hard failure if: - Invalid port is provided (outside range 1-65535); or - The same server is added to inventory again (duplicate). +##### An example of usage + +```lua +local k = require('kafe') +local username = k.getenv('USER') -- get username from env? :) +k.add_inventory(username, 'one.example.org', 22, 'staging', 'example') +k.add_inventory(username, 'two.example.org', 22, 'production', 'example') +k.add_inventory(username, 'three.example.org', 22, 'production', 'example') +``` + ### bool k.on(string role, function callable [,bool skip_empty_inv = true]) Execute given function on each remote server with given role, in current environment. @@ -102,6 +182,23 @@ Results in hard failure if: - Called in a way that results in nested context; or - If there are no servers in given inventory and `skip_empty_env` is not true. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + local my_todo = function() + -- ... your code here + end + + -- Execute function my_todo on all servers with role example_role + -- Fail the script if execution failed for any reason + if not k.on('example_role', my_todo) + then error('Could not execute my_todo for some reason') end +end) +``` + ### void k.within(string directory_path) Execute all subsequent *remote* commands in given context within given directory. @@ -112,6 +209,21 @@ Effectively prepends `cd &&` to all subsequent shell commands in the same subsequent remote shell commands in directory that does not exist will result in their hard failure. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + local my_todo = function() + k.within('/tmp') + k.shell('ls') -- will print contents of remote /tmp in stdout + end + + k.on('example_role', my_todo) +end) +``` + ### (string stdout, string stderr, int exit_code) k.exec(string command [, bool print_output = true]) Execute a remote shell command and return it's outputs along with exit code. @@ -119,11 +231,43 @@ Execute a remote shell command and return it's outputs along with exit code. Second optional argument indicates if the remote output should also be logged in output of the tool. This option is enabled by default. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + local my_todo = function() + local out, err, code = k.exec('whoami') + -- out: remote stdout + -- err: remote stderr + -- code: remote exit code + end + + k.on('example_role', my_todo) +end) +``` + ### bool k.shell(string command) Execute a remote shell command, log output and return exit status as boolean. Will return true if exit status of the remote command is `0`, false otherwise. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + local my_todo = function() + local result = k.shell('whoami') + -- result: boolean true if whoami exit code == 0, false otherwise + end + + k.on('example_role', my_todo) +end) +``` + ### string k.archive_dir_tmp(string directory) Create a `.tar.gz` archive from given *local* directory and get the full path to the archive once created. @@ -135,6 +279,17 @@ Results in hard failure if: - Archive directory does not exist. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + local archive = k.archive_dir_tmp('/home/example/some_folder') + -- archive: string path to .tar.gz file +end) +``` + ### void k.archive_dir(string directory, string archive_file) Create a `.tar.gz` archive from given *local* directory and get the full path to the archive oncre created. @@ -145,6 +300,16 @@ Results in hard failure if: - Archive directory does not exist; or - File or directory exists at the path provided in `archive_file`. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + k.archive_dir_tmp('/home/example/some_folder', '/home/example/some_archive.tar.gz') +end) +``` + ### bool k.upload_file(string local_file, string remote_file) Upload local file to remote server in given path. `remote_file` can be @@ -156,6 +321,26 @@ This command returns true if upload succeeded, and false on failure. **IMPORTANT:** any existing remote files will be silently overwritten. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + local archive = k.archive_dir_tmp('/home/example/some_folder') + + local my_todo = function() + if not k.shell('mkdir -p /tmp/example') + then error('Failed to ensure remote directory') end + + if not k.upload_file(archive, '/tmp/example/') + then error('Failed to upload archive to remote directory') end + end + + k.on('example_role', my_todo) +end) +``` + ### bool k.download_file(string local_file, string remote_file) Download remote file from remote server to given local path. @@ -164,6 +349,21 @@ This command returns true if download succeeded, and false on failure. **IMPORTANT:** any existing local files will be silently overwritten. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + local my_todo = function() + if not k.download_file('/local/path/example.txt', '/remote/path/example.txt') + then error('Failed to download remote file') end + end + + k.on('example_role', my_todo) +end) +``` + ### bool k.upload_str(string content, string remote_file) Upload text as file to remote server in given path. `remote_file` must be valid file. @@ -174,12 +374,43 @@ This command returns true if upload succeeded, and false on failure. **IMPORTANT:** any existing remote files will be silently overwritten. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + local my_todo = function() + if not k.upload_str('Hello world!', '/remote/path/example.txt') + then error('Failed to upload text to remote file') end + end + + k.on('example_role', my_todo) +end) +``` + ### string|bool k.download_str(string remote_file) Download remote file from remote server as string. This command returns content of the file as string if download succeeded, and false on failure. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + local my_todo = function() + local content = k.download_str('/remote/path/example.txt') + if not content then error('Failed to download remote file as text') end + print(content) -- Prints contents of the remote file to stdout + end + + k.on('example_role', my_todo) +end) +``` + ## void k.define(string key, any value) Define a runtime variable in context of the executing script. These @@ -192,6 +423,25 @@ Accepts any value that can be cast to string as second argument. **IMPORTANT:** existing values using the same key will be silently overwritten. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + k.define('remote_path', '/tmp/example') + + local my_todo = function() + -- Try to create /tmp/example/ using placeholder + if not k.shell('mkdir -p {{remote_path}}') + -- ... or fail with meaningful error + then error('Could not create remote path {{remote_path}}') end + end + + k.on('example_role', my_todo) +end) +``` + ## string k.strfvars(string text) Replace any placeholder values of format `{{key}}` to their values as defined @@ -201,6 +451,17 @@ using `k.define(...)`. **IMPORTANT:** non-existent values will be replaced with empty string. +##### An example of usage + +```lua +local k = require('kafe') + +k.define('remote_path', '/tmp/example') +k.define('remote_file', 'hello.txt') +local path = k.strfvars('{{remote_path}}/{{remote_file}}') +print(path) -- prints /tmp/example/hello.txt +``` + ## string k.strfenv(string text) Replace any placeholder values of format `{{key}}` to their values from @@ -210,18 +471,71 @@ executing environment variables. **IMPORTANT:** non-existent values will be replaced with empty string. +**IMPORTANT:** script environment and Lua standard `os.getenv(...)` MIGHT return different results when executing +environment is not CLI (future compatibility). You should always use `k.strfenv` and `k.getenv` to access +environment variables from within Kafe scripts. + +##### An example of usage + +```lua +local k = require('kafe') + +local hello = k.strfvars('Hello, my name is {{USER}}!') +print(hello) -- prints Hello, my name is ! +``` + +## string|nil k.getenv(string key) + +Get an environment value. Returns nil if environment value is not defined. + +**IMPORTANT:** keys are case sensitive. + +**IMPORTANT:** script environment and Lua standard `os.getenv(...)` MIGHT return different results when executing +environment is not CLI (future compatibility). You should always use `k.strfenv` and `k.getenv` to access +environment variables from within Kafe scripts. + +##### An example of usage + +```lua +local k = require('kafe') + +local user = k.getenv('USER') +print(user) -- prints +``` + ## (string stdout, int code) k.local_exec(string command [, bool print_output = true]) Execute local shell command and return it's stdout and exit code. Second optional argument indicates if the remote output should also be logged in output of the tool. This option is enabled by default. +```lua +local k = require('kafe') + +k.task('example_task', function() + local out, err, code = k.local_exec('whoami') + -- out: local stdout + -- err: local stderr + -- code: local exit code +end) +``` ## bool k.local_shell(string command) Execute a local shell command, log output and return exit status as boolean. Will return true if exit status of the local command is `0`, false otherwise. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + local result = k.local_shell('whoami') + -- result: boolean true if whoami exit code == 0, false otherwise +end) +``` + ## void k.local_within(string directory_path) Execute all subsequent *local* commands within given directory, regardless of the remote context. @@ -230,4 +544,15 @@ Results in hard failure if: - Given directory does not exist. +##### An example of usage + +```lua +local k = require('kafe') + +k.task('example_task', function() + k.local_within('/tmp') + k.local_shell('ls') -- will print contents of local /tmp in stdout +end) +``` + diff --git a/examples/deploy.lua b/examples/deploy.lua index 439c7ef..cff6969 100644 --- a/examples/deploy.lua +++ b/examples/deploy.lua @@ -86,4 +86,4 @@ k.task('deploy', function() if not k.on('example_app', reload_service) then error('Failed to reload the service') end -end \ No newline at end of file +end) diff --git a/examples/deploy_from_git.lua b/examples/deploy_from_git.lua index 67490f0..7590077 100644 --- a/examples/deploy_from_git.lua +++ b/examples/deploy_from_git.lua @@ -107,4 +107,4 @@ k.task('deploy', function() if not k.on('example_app', reload_service) then error('Failed to reload the service') end -end \ No newline at end of file +end) diff --git a/libkafe/include/kafe/context.hpp b/libkafe/include/kafe/context.hpp index 4cce1df..1870d75 100644 --- a/libkafe/include/kafe/context.hpp +++ b/libkafe/include/kafe/context.hpp @@ -43,7 +43,7 @@ namespace kafe { virtual ~Context(); - [[nodiscard]] const map &get_envvals() const; + [[nodiscard]] const map *get_envvals() const; [[nodiscard]] const string &get_environment() const; diff --git a/libkafe/include/kafe/execution_scope.hpp b/libkafe/include/kafe/execution_scope.hpp index 06fc414..a84562d 100644 --- a/libkafe/include/kafe/execution_scope.hpp +++ b/libkafe/include/kafe/execution_scope.hpp @@ -44,10 +44,14 @@ namespace kafe { vector files_to_rm_on_destruct; map values; LocalApi *local; + const vector &extra_args; + const string& project_file; public: explicit ExecutionScope( const Context &context, - const Inventory &inventory + const Inventory &inventory, + const vector &extra_args, + const string& project_file ); [[nodiscard]] const Context *get_context() const; @@ -79,6 +83,12 @@ namespace kafe { [[nodiscard]] string replace_env(const string &input) const; [[nodiscard]] LocalApi *get_local_api() const; + + const vector &get_extra_args() const; + + const string &get_project_file() const; + + const string get_env(const string &key) const; }; } diff --git a/libkafe/include/kafe/logging.hpp b/libkafe/include/kafe/logging.hpp index c802904..2ed971c 100644 --- a/libkafe/include/kafe/logging.hpp +++ b/libkafe/include/kafe/logging.hpp @@ -74,15 +74,16 @@ namespace kafe { string unit; double duration; - if (duration_micros < 1000) { - unit = "μs"; - duration = (double) duration_micros; - } else if (duration_micros < 100000) { + + if (duration_micros > 1000000) { + unit = "s"; + duration = (double) duration_micros / 1000000; + } else if (duration_micros > 1000) { unit = "ms"; duration = (double) duration_micros / 1000; - } else if (duration_micros < 1000000) { - unit = "s"; - duration = (double) duration_micros / 1000 / 1000; + } else { + unit = "μs"; + duration = (double) duration_micros; } buffer << duration << unit; diff --git a/libkafe/include/kafe/project.hpp b/libkafe/include/kafe/project.hpp index 64bbafa..f606822 100644 --- a/libkafe/include/kafe/project.hpp +++ b/libkafe/include/kafe/project.hpp @@ -47,7 +47,8 @@ namespace kafe { void execute( const Context &context, - const Inventory &inventory + const Inventory &inventory, + const vector &extra_args ); }; } diff --git a/libkafe/include/kafe/remote/ssh_api.hpp b/libkafe/include/kafe/remote/ssh_api.hpp index 543f73a..1340a71 100644 --- a/libkafe/include/kafe/remote/ssh_api.hpp +++ b/libkafe/include/kafe/remote/ssh_api.hpp @@ -56,7 +56,7 @@ namespace kafe::remote { void scp_download_file(const string &file, const string &remote_path) const; - string scp_download_file_as_string(const string &remote_file) const; + [[nodiscard]] string scp_download_file_as_string(const string &remote_file) const; void scp_upload_file_from_string(const string &content, const string &remote_file) const; }; diff --git a/libkafe/include/kafe/remote/ssh_manager.hpp b/libkafe/include/kafe/remote/ssh_manager.hpp index f808a64..76f63ec 100644 --- a/libkafe/include/kafe/remote/ssh_manager.hpp +++ b/libkafe/include/kafe/remote/ssh_manager.hpp @@ -32,9 +32,10 @@ using namespace kafe::project; namespace kafe::remote { class SshManager { SshPool *pool; + const map *envvals; const InventoryItem *item; public: - SshManager(const SshPool *pool, const InventoryItem *item); + SshManager(const SshPool *pool, const map *envvals, const InventoryItem *item); const SshSession *get_or_create_session(LogLevel level); }; diff --git a/libkafe/include/kafe/remote/ssh_pool.hpp b/libkafe/include/kafe/remote/ssh_pool.hpp index c4bf87b..0f287d0 100644 --- a/libkafe/include/kafe/remote/ssh_pool.hpp +++ b/libkafe/include/kafe/remote/ssh_pool.hpp @@ -27,7 +27,6 @@ using namespace std; namespace kafe::remote { - // TODO: expire sessions? class SshPool { map sessions = {}; diff --git a/libkafe/include/kafe/remote/ssh_session.hpp b/libkafe/include/kafe/remote/ssh_session.hpp index 9aed816..59ffe89 100644 --- a/libkafe/include/kafe/remote/ssh_session.hpp +++ b/libkafe/include/kafe/remote/ssh_session.hpp @@ -28,6 +28,7 @@ extern "C" { #endif #include +#include #include "kafe/runtime/runtime_exception.hpp" #include "kafe/logging.hpp" @@ -43,7 +44,7 @@ namespace kafe::remote { class SshSession { ssh_session session; public: - SshSession(const string &user, const string &host, int port, LogLevel level); + SshSession(const map *envvals, const string &user, const string &host, unsigned int port, LogLevel level); [[nodiscard]] bool is_active() const; diff --git a/libkafe/include/kafe/runtime/runtime_exception.hpp b/libkafe/include/kafe/runtime/runtime_exception.hpp index f72d71e..9b54b0c 100644 --- a/libkafe/include/kafe/runtime/runtime_exception.hpp +++ b/libkafe/include/kafe/runtime/runtime_exception.hpp @@ -27,15 +27,11 @@ using namespace std; namespace kafe::runtime { class RuntimeException : public exception { - string message; + char* message; public: - RuntimeException(); - explicit RuntimeException(const char *format, ...); - explicit RuntimeException(const string *format, ...); - [[nodiscard]] const char *what() const noexcept override; }; } diff --git a/libkafe/include/kafe/scripting/script.hpp b/libkafe/include/kafe/scripting/script.hpp index 45599b2..b458f6b 100644 --- a/libkafe/include/kafe/scripting/script.hpp +++ b/libkafe/include/kafe/scripting/script.hpp @@ -55,7 +55,7 @@ namespace kafe::scripting { void evaluate(); - void invoke_function_by_ref(int reference); + void invoke_function_by_ref(int reference, const vector &extra_args); private: void initialize(); diff --git a/libkafe/src/context.cpp b/libkafe/src/context.cpp index 9b2359f..323ba4c 100644 --- a/libkafe/src/context.cpp +++ b/libkafe/src/context.cpp @@ -43,7 +43,7 @@ namespace kafe { return log_listener; } - const map &Context::get_envvals() const { - return envvals; + const map *Context::get_envvals() const { + return &envvals; } } \ No newline at end of file diff --git a/libkafe/src/execution_scope.cpp b/libkafe/src/execution_scope.cpp index c44b4b1..b600bec 100644 --- a/libkafe/src/execution_scope.cpp +++ b/libkafe/src/execution_scope.cpp @@ -29,8 +29,10 @@ using namespace kafe::remote; namespace kafe { ExecutionScope::ExecutionScope( const Context &context, - const Inventory &inventory - ) : context(context), inventory(inventory) { + const Inventory &inventory, + const vector &extra_args, + const string &project_file + ) : context(context), inventory(inventory), extra_args(extra_args), project_file(project_file) { this->ssh_pool = new SshPool(); this->tasks = new TaskList(); this->local = new LocalApi(context.get_log_listener()); @@ -127,7 +129,7 @@ namespace kafe { } string ExecutionScope::replace_env(const string &input) const { - auto envvals = context.get_envvals(); + const auto *envvals = context.get_envvals(); ostringstream out; size_t pos = 0; for (;;) { @@ -141,9 +143,9 @@ namespace kafe { out.write(&*input.begin() + pos, subst_pos - pos); subst_pos += STRLEN_START_TOK; const char *key = input.substr(subst_pos, end_pos - subst_pos).c_str(); - auto env_val = envvals.find(key); + auto env_val = envvals->find(key); - if (env_val != envvals.end()) { + if (env_val != envvals->end()) { out << env_val->second; } @@ -154,7 +156,27 @@ namespace kafe { return out.str(); } + const string ExecutionScope::get_env(const string &key) const { + const auto *envvals = context.get_envvals(); + + auto env_val = envvals->find(key); + + if (env_val != envvals->end()) { + return env_val->second; + } + + return {}; + } + LocalApi *ExecutionScope::get_local_api() const { return local; } + + const vector &ExecutionScope::get_extra_args() const { + return extra_args; + } + + const string &ExecutionScope::get_project_file() const { + return project_file; + } } \ No newline at end of file diff --git a/libkafe/src/io/archive.cpp b/libkafe/src/io/archive.cpp index b6908eb..8cfcb17 100644 --- a/libkafe/src/io/archive.cpp +++ b/libkafe/src/io/archive.cpp @@ -32,14 +32,13 @@ namespace kafe::io { static const int ARCHIVE_FILE_BUFFER_S = 4096; string Archive::tmp_archive_from_directory(const string &directory) { - auto name = tmpnam(nullptr); // TODO replace + auto *name = tmpnam(nullptr); // TODO replace auto upload_name = string(name) + ".tar.gz"; archive_from_directory(upload_name, directory); return upload_name; } // TODO: implement .kafeignore - // TODO: https://stackoverflow.com/questions/23330065/adding-directory-to-tarfile-with-libarchive void Archive::archive_from_directory(const string &archive_path, const string &directory) { if (!FileSystem::is_directory(directory)) { throw RuntimeException("Can not create archive - directory <%s> not found", directory.c_str()); @@ -70,7 +69,7 @@ namespace kafe::io { while (iter != end) { auto path_abs = std_fs::absolute(iter->path()); - if (std_fs::is_directory(path_abs)) { + if (!FileSystem::is_file_or_symlink(path_abs) && !FileSystem::is_directory(path_abs)) { ++iter; continue; } @@ -83,20 +82,33 @@ namespace kafe::io { } // ENDTODO - auto size = std_fs::file_size(path_abs); + auto is_dir = std_fs::is_directory(path_abs); + size_t size = 0; + if (!is_dir) { + size = std_fs::file_size(path_abs); + } struct stat buf{}; stat(path_abs.c_str(), &buf); auto *entry = archive_entry_new(); archive_entry_set_pathname(entry, path_rel.c_str()); - archive_entry_set_size(entry, size); - archive_entry_set_filetype(entry, AE_IFREG); + archive_entry_set_filetype(entry, is_dir ? AE_IFDIR : AE_IFREG); archive_entry_set_perm(entry, buf.st_mode); -// archive_entry_set_mtime(entry, buf.st_mtim.tv_sec, 0); - archive_write_header(archive, entry); archive_entry_copy_stat(entry, &buf); + if (!is_dir) { + archive_entry_set_size(entry, size); + } + archive_write_header(archive, entry); + + if (std_fs::is_directory(path_abs)) { + archive_write_finish_entry(archive); + archive_entry_free(entry); + ++iter; + continue; + } + ifstream fin(path_abs.string(), ifstream::binary); char buffer[ARCHIVE_FILE_BUFFER_S]; do { @@ -105,6 +117,8 @@ namespace kafe::io { } while (fin); fin.close(); + + archive_write_finish_entry(archive); archive_entry_free(entry); ++iter; diff --git a/libkafe/src/local/local_api.cpp b/libkafe/src/local/local_api.cpp index fabe1ee..c3151ee 100644 --- a/libkafe/src/local/local_api.cpp +++ b/libkafe/src/local/local_api.cpp @@ -114,7 +114,7 @@ namespace kafe::local { FILE *p = ::popen(cmd.c_str(), "r"); if (nullptr == p) { - return LocalShellResult("", -1); + return LocalShellResult({}, -1); } auto output = read_out(p, print_output); diff --git a/libkafe/src/project.cpp b/libkafe/src/project.cpp index fc6138c..9783ac8 100644 --- a/libkafe/src/project.cpp +++ b/libkafe/src/project.cpp @@ -38,16 +38,20 @@ namespace kafe { void Project::execute( const Context &context, - const Inventory &inventory + const Inventory &inventory, + const vector &extra_args ) { - auto scope = ExecutionScope(context, inventory); + auto main_file = FileSystem::absolute(project_file, std_fs::current_path()).string(); + auto scope = ExecutionScope(context, inventory, extra_args, main_file); LoggingTimer timer; auto script = Script(scope); auto *logger = const_cast(scope.get_context()->get_log_listener()); + logger->emit_debug("Resolved project file is <%s>", main_file.c_str()); + timer = logger->emit_info_wt("Loading project file <%s>", project_file.c_str()); - script.load_file(project_file); + script.load_file(main_file); logger->emit_success(&timer, "Loaded project file <%s>", project_file.c_str()); timer = logger->emit_info_wt("Evaluating project file <%s>", project_file.c_str()); @@ -55,21 +59,21 @@ namespace kafe { logger->emit_success(&timer, "Done evaluating project file <%s>", project_file.c_str()); logger->emit_info("Verifying all tasks requested are defined"); - for (auto &task_name : scope.get_context()->get_tasks()) { + for (const auto &task_name : scope.get_context()->get_tasks()) { if (!scope.get_tasks()->task_exists(task_name)) { throw UnknownTaskException("Task <%s> is not defined in project", task_name.c_str()); } } logger->emit_success("Tasks verified"); - for (auto &task_name : scope.get_context()->get_tasks()) { + for (const auto &task_name : scope.get_context()->get_tasks()) { timer = logger->emit_info_wt("Executing task <%s>", task_name.c_str()); logger->context_push(task_name); const Task *task = scope.get_tasks()->get_task(task_name); int ref = task->get_function_reference(); - script.invoke_function_by_ref(ref); + script.invoke_function_by_ref(ref, extra_args); logger->context_pop(); logger->emit_success(&timer, "Task <%s> completed", task_name.c_str()); diff --git a/libkafe/src/project/inventory.cpp b/libkafe/src/project/inventory.cpp index 7250ff9..62893bb 100644 --- a/libkafe/src/project/inventory.cpp +++ b/libkafe/src/project/inventory.cpp @@ -104,7 +104,7 @@ namespace kafe::project { } Inventory::~Inventory() { - for (auto item : items) { + for (const auto *item : items) { delete (item); } items.clear(); diff --git a/libkafe/src/remote/ssh_api.cpp b/libkafe/src/remote/ssh_api.cpp index c15fc6d..a086bc1 100644 --- a/libkafe/src/remote/ssh_api.cpp +++ b/libkafe/src/remote/ssh_api.cpp @@ -55,7 +55,7 @@ namespace kafe::remote { char *output = (char *) malloc(output_size); unsigned long current_pos = 0; unsigned long last_line_pos = 0; - int ssh_n_read; + int ssh_n_read = 0; char buffer[buffer_size]; char *line_buffer = (char *) malloc(256); do { @@ -113,8 +113,8 @@ namespace kafe::remote { } RemoteResult SshApi::execute(const string &command, const bool print_output) const { - auto session = manager->get_or_create_session(log_listener->get_level()); - auto ssh_session = session->get_ssh_session(); + const auto *session = manager->get_or_create_session(log_listener->get_level()); + auto *ssh_session = session->get_ssh_session(); ssh_channel channel; int rc; @@ -140,7 +140,7 @@ namespace kafe::remote { "In directory <%s> executing <%s>", this->current_chdir.c_str(), command.c_str()); cmd_buf << "cd " << this->current_chdir << " && "; } else { - timer = log_listener->emit_info_wt("Executing command %s", command.c_str()); + timer = log_listener->emit_info_wt("Executing command <%s>", command.c_str()); } cmd_buf << command; @@ -195,11 +195,10 @@ namespace kafe::remote { remote_dir = remote_file_path; } - auto session = manager->get_or_create_session(log_listener->get_level()); - auto ssh_session = session->get_ssh_session(); - - auto scp = ssh_scp_new(ssh_session, SSH_SCP_WRITE, remote_dir.c_str()); + const auto *session = manager->get_or_create_session(log_listener->get_level()); + auto *ssh_session = session->get_ssh_session(); + auto *scp = ssh_scp_new(ssh_session, SSH_SCP_WRITE, remote_dir.c_str()); if (nullptr == scp) { throw RuntimeException("SCP error [code %d: %s]", ssh_get_error_code(ssh_session), ssh_get_error(ssh_session)); @@ -253,11 +252,10 @@ namespace kafe::remote { remote_dir = remote_file_path; } - auto session = manager->get_or_create_session(log_listener->get_level()); - auto ssh_session = session->get_ssh_session(); - - auto scp = ssh_scp_new(ssh_session, SSH_SCP_WRITE, remote_dir.c_str()); + const auto *session = manager->get_or_create_session(log_listener->get_level()); + auto *ssh_session = session->get_ssh_session(); + auto *scp = ssh_scp_new(ssh_session, SSH_SCP_WRITE, remote_dir.c_str()); if (nullptr == scp) { throw RuntimeException("SCP error [code %d: %s]", ssh_get_error_code(ssh_session), ssh_get_error(ssh_session)); @@ -288,10 +286,10 @@ namespace kafe::remote { } void SshApi::scp_download_file(const string &file, const string &remote_file) const { - auto session = manager->get_or_create_session(log_listener->get_level()); - auto ssh_session = session->get_ssh_session(); + const auto *session = manager->get_or_create_session(log_listener->get_level()); + auto *ssh_session = session->get_ssh_session(); - auto scp = ssh_scp_new(ssh_session, SSH_SCP_READ, remote_file.c_str()); + auto *scp = ssh_scp_new(ssh_session, SSH_SCP_READ, remote_file.c_str()); if (scp == nullptr) { throw RuntimeException("SSH error: %s", ssh_get_error(ssh_session)); } @@ -313,7 +311,7 @@ namespace kafe::remote { ssh_scp_accept_request(scp); ofstream out(file, ofstream::binary); - const int bsize = size > 4096 ? 4096 : (int) size; + const size_t bsize = size > 4096 ? 4096 : (int) size; char *buffer = (char *) malloc(bsize); int read_count; do { @@ -331,10 +329,10 @@ namespace kafe::remote { } string SshApi::scp_download_file_as_string(const string &remote_file) const { - auto session = manager->get_or_create_session(log_listener->get_level()); - auto ssh_session = session->get_ssh_session(); + const auto *session = manager->get_or_create_session(log_listener->get_level()); + auto *ssh_session = session->get_ssh_session(); - auto scp = ssh_scp_new(ssh_session, SSH_SCP_READ, remote_file.c_str()); + auto *scp = ssh_scp_new(ssh_session, SSH_SCP_READ, remote_file.c_str()); if (scp == nullptr) { throw RuntimeException("SSH error: %s", ssh_get_error(ssh_session)); } @@ -351,14 +349,14 @@ namespace kafe::remote { throw RuntimeException("SSH error: %s", ssh_get_error(ssh_session)); } - auto size = ssh_scp_request_get_size(scp); + const size_t size = ssh_scp_request_get_size(scp); ssh_scp_accept_request(scp); stringstream bfs; - const int bsize = size > 4096 ? 4096 : (int) size; + const size_t bsize = size > 4096 ? 4096 : (size_t) size; char *buffer = (char *) malloc(bsize); - int read_count; + int read_count = 0; do { read_count = ssh_scp_read(scp, buffer, bsize); if (read_count > 0) { diff --git a/libkafe/src/remote/ssh_manager.cpp b/libkafe/src/remote/ssh_manager.cpp index f12a954..6ac9c49 100644 --- a/libkafe/src/remote/ssh_manager.cpp +++ b/libkafe/src/remote/ssh_manager.cpp @@ -20,15 +20,15 @@ #include "kafe/remote/ssh_manager.hpp" namespace kafe::remote { - SshManager::SshManager(const SshPool *pool, const InventoryItem *item) - : pool(const_cast(pool)), item(item) { + SshManager::SshManager(const SshPool *pool, const map *envvals, const InventoryItem *item) + : pool(const_cast(pool)), envvals(envvals), item(item) { } const SshSession *SshManager::get_or_create_session(LogLevel level) { auto remote_id = item->remote_id(); if (pool->has_session(remote_id)) { - auto current_session = pool->get_session(remote_id); + auto *current_session = pool->get_session(remote_id); if (current_session->is_active()) { return current_session; @@ -37,7 +37,8 @@ namespace kafe::remote { pool->remove_session(remote_id); } - auto session = new SshSession( + auto *session = new SshSession( + envvals, item->get_user(), item->get_host(), item->get_port(), diff --git a/libkafe/src/remote/ssh_session.cpp b/libkafe/src/remote/ssh_session.cpp index 3856e19..975243a 100644 --- a/libkafe/src/remote/ssh_session.cpp +++ b/libkafe/src/remote/ssh_session.cpp @@ -20,16 +20,15 @@ #include "kafe/remote/ssh_session.hpp" namespace kafe::remote { - SshSession::SshSession(const string &user, const string &host, int port, LogLevel level) { - ssh_session session_new = ssh_new(); + SshSession::SshSession(const map *envvals, const string &user, const string &host, + unsigned int port, LogLevel level) { + ssh_session session_new = this->session = ssh_new(); - int verbosity; + int verbosity = SSH_LOG_NOLOG; if (level == LogLevel::ALL) { verbosity = SSH_LOG_FUNCTIONS; } else if (level == LogLevel::TRACE) { verbosity = SSH_LOG_PROTOCOL; - } else { - verbosity = SSH_LOG_NOLOG; } ssh_options_set(session_new, SSH_OPTIONS_LOG_VERBOSITY, &verbosity); @@ -40,23 +39,90 @@ namespace kafe::remote { auto result = ssh_connect(session_new); if (SSH_OK != result) { - throw SshSessionException(); + const auto *error = ssh_get_error(session_new); + throw SshSessionException("Remote connection failed. %s", error); } +#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0, 8, 4) + auto is_known = ssh_session_is_known_server(session_new); + auto is_known_b = SSH_KNOWN_HOSTS_OK == is_known; +#else auto is_known = ssh_is_server_known(session_new); - if (is_known != SSH_SERVER_KNOWN_OK) { - throw SshSessionException(); + auto is_known_b = SSH_SERVER_KNOWN_OK == is_known; +#endif + if (!is_known_b) { + throw SshSessionException( + "Cowardly refusing to connect to <%s:%d> - remote host key verification failed. " + "You are required to verify this host and add it to known hosts independently before " + "any attempts to connect to it using Kafe.", + host.c_str(), + port + ); } - // TODO: load passphrase from env properties - auto is_auth = ssh_userauth_publickey_auto(session_new, nullptr, nullptr); + string key_passphrase; + auto env_key_passphrase = envvals->find("KAFE_SSH_PKEY_PASS"); + if (env_key_passphrase != envvals->end()) { + key_passphrase = env_key_passphrase->second; + } + + string user_passphrase; + auto env_user_passphrase = envvals->find("KAFE_SSH_USER_PASS"); + if (env_user_passphrase != envvals->end()) { + user_passphrase = env_user_passphrase->second; + } + + auto req_retr_auth_none = ssh_userauth_none(session_new, nullptr); + + if (SSH_AUTH_ERROR == req_retr_auth_none) { + throw SshSessionException("Failed to fetch list of methods for host <%s:%d>", host.c_str(), port); + } + + unsigned int methods = ssh_userauth_list(session_new, nullptr); + int is_auth; + + if (SSH_AUTH_METHOD_UNKNOWN == methods) { + // Fallback to public key IF no method listed + is_auth = ssh_userauth_publickey_auto( + session_new, + nullptr, + key_passphrase.empty() ? nullptr : key_passphrase.c_str() + ); + + if (SSH_AUTH_SUCCESS == is_auth) { + return; + } + + throw SshSessionException("Authentication failed for host <%s:%d>.", host.c_str(), port); + } + + if (methods & (unsigned int) SSH_AUTH_METHOD_PUBLICKEY) { + is_auth = ssh_userauth_publickey_auto( + session_new, + nullptr, + key_passphrase.empty() ? nullptr : key_passphrase.c_str() + ); + if (SSH_AUTH_SUCCESS == is_auth) { + return; + } + } + + if (methods & (unsigned int) SSH_AUTH_METHOD_GSSAPI_MIC) { + is_auth = ssh_userauth_gssapi(session_new); + if (SSH_AUTH_SUCCESS == is_auth) { + return; + } + } - // TODO: verify - if (is_auth != SSH_AUTH_SUCCESS) { - throw SshSessionException(); + if ((methods & (unsigned int) SSH_AUTH_METHOD_PASSWORD) && !user_passphrase.empty()) { + is_auth = ssh_userauth_password(session_new, nullptr, user_passphrase.c_str()); + if (SSH_AUTH_SUCCESS == is_auth) { + return; + } } - this->session = session_new; + const auto *error = ssh_get_error(session_new); + throw SshSessionException("Authentication failed for host <%s:%d>. %s", host.c_str(), port, error); } SshSession::~SshSession() { @@ -77,5 +143,6 @@ namespace kafe::remote { if (ssh_is_connected(session)) { ssh_disconnect(session); } + ssh_free(session); } } \ No newline at end of file diff --git a/libkafe/src/runtime/runtime_exception.cpp b/libkafe/src/runtime/runtime_exception.cpp index 60e3b4c..373b8f2 100644 --- a/libkafe/src/runtime/runtime_exception.cpp +++ b/libkafe/src/runtime/runtime_exception.cpp @@ -22,10 +22,6 @@ #include "kafe/runtime/runtime_exception.hpp" namespace kafe::runtime { - RuntimeException::RuntimeException() { - message = exception::what(); - } - RuntimeException::RuntimeException(const char *format, ...) { char buffer[4096]; va_list args; @@ -35,16 +31,7 @@ namespace kafe::runtime { this->message = buffer; } - RuntimeException::RuntimeException(const string *format, ...) { - char buffer[4096]; - va_list args; - va_start(args, format); - vsnprintf(buffer, 4096, format->c_str(), args); - va_end(args); - this->message = buffer; - } - const char *RuntimeException::what() const noexcept { - return message.c_str(); + return message; } } \ No newline at end of file diff --git a/libkafe/src/scripting/script.cpp b/libkafe/src/scripting/script.cpp index 70fc9e3..076bf26 100644 --- a/libkafe/src/scripting/script.cpp +++ b/libkafe/src/scripting/script.cpp @@ -18,7 +18,6 @@ */ #include -#include #include "kafe/version.hpp" #include "kafe/execution_scope.hpp" @@ -30,7 +29,6 @@ using namespace kafe::io; -static mutex state_lock; static map state; static inline ExecutionScope *get_scope(lua_State *L) { @@ -43,20 +41,18 @@ static inline ExecutionScope *get_scope(lua_State *L) { return const_cast(res->second); } -// TODO - review error messages namespace kafe::scripting { - // Lua stdout/stderr // TODO: split line by line static int lua_logger_print(lua_State *L, bool is_err) { - auto scope = get_scope(L); + auto *scope = get_scope(L); int n_args = lua_gettop(L); lua_getglobal(L, "tostring"); for (int i = 1; i <= n_args; i++) { lua_pushvalue(L, -1); lua_pushvalue(L, i); lua_call(L, 1, 1); - auto line = lua_tostring(L, -1); + const auto *line = lua_tostring(L, -1); if (nullptr == line) { return luaL_error(L, "'tostring' must return a string to 'print'"); } @@ -83,9 +79,7 @@ namespace kafe::scripting { // end Lua stdout/stderr Script::Script(const ExecutionScope &scope) : scope(scope) { - state_lock.lock(); - - auto lst = luaL_newstate(); + auto *lst = luaL_newstate(); auto entry = pair(lst, &scope); state.insert(entry); @@ -97,23 +91,20 @@ namespace kafe::scripting { lua_register(lua_state, "print_err", lua_logger_print_err); initialize(); - - state_lock.unlock(); } Script::~Script() { - state_lock.lock(); state.erase(this->lua_state); lua_close(this->lua_state); - state_lock.unlock(); } void Script::load_file(const string &script_file) { auto status = luaL_loadfile(this->lua_state, script_file.c_str()); if (status) { - auto lua_error = lua_tostring(this->lua_state, -1); - throw ScriptEngineException("%s", lua_error); + const auto *lua_error = lua_tostring(this->lua_state, -1); + const auto with_vars = scope.replace_vars(lua_error); + throw ScriptEngineException("%s", with_vars.c_str()); } } @@ -121,18 +112,25 @@ namespace kafe::scripting { int status = lua_pcall(this->lua_state, 0, LUA_MULTRET, 0); if (status) { - auto lua_error = lua_tostring(this->lua_state, -1); - throw ScriptEvaluationException("%s", lua_error); + const auto *lua_error = lua_tostring(this->lua_state, -1); + const auto with_vars = scope.replace_vars(lua_error); + throw ScriptEvaluationException("%s", with_vars.c_str()); } } - void Script::invoke_function_by_ref(const int reference) { + void Script::invoke_function_by_ref(const int reference, const vector &extra_args) { lua_rawgeti(this->lua_state, LUA_REGISTRYINDEX, reference); - int status = lua_pcall(this->lua_state, 0, 0, 0); + + for (const auto &var : extra_args) { + lua_pushstring(this->lua_state, var.c_str()); + } + + unsigned long status = lua_pcall(this->lua_state, extra_args.size(), 0, 0); if (status) { - auto lua_error = lua_tostring(this->lua_state, -1); - throw ScriptInvocationException("%s", lua_error); + const auto *lua_error = lua_tostring(this->lua_state, -1); + const auto with_vars = scope.replace_vars(lua_error); + throw ScriptInvocationException("%s", with_vars.c_str()); } } @@ -164,7 +162,7 @@ namespace kafe::scripting { // TODO: allow kDSN format - int lua_api_inventory_add(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (5 != lua_gettop(L)) { return luaL_error(L, "Expected five arguments - username, hostname, port, environment, role"); @@ -190,19 +188,18 @@ namespace kafe::scripting { return luaL_error(L, "Argument five must be a string - role"); } - auto username = luaL_checkstring(L, 1); - auto hostname = luaL_checkstring(L, 2); + const auto *username = luaL_checkstring(L, 1); + const auto *hostname = luaL_checkstring(L, 2); auto port = luaL_checkinteger(L, 3); - auto environment = luaL_checkstring(L, 4); - auto role = luaL_checkstring(L, 5); + const auto *environment = luaL_checkstring(L, 4); + const auto *role = luaL_checkstring(L, 5); if (port < 1 || port > 65535) { return luaL_error(L, "Invalid port value <%s>", port); } - auto item = new InventoryItem(username, hostname, (unsigned int) port, environment, role); - - auto inventory = const_cast(scope->get_inventory()); + auto *item = new InventoryItem(username, hostname, (unsigned int) port, environment, role); + auto *inventory = const_cast(scope->get_inventory()); if (inventory->item_exists(*item)) { return luaL_error(L, "Duplicate inventory item <%s>", item->to_string().c_str()); @@ -223,7 +220,7 @@ namespace kafe::scripting { } int lua_api_task_define(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (2 != lua_gettop(L)) { return luaL_error(L, "Expected two arguments, task name and task function"); @@ -237,11 +234,11 @@ namespace kafe::scripting { return luaL_error(L, "Argument two must be a function"); } - auto task_name = luaL_checkstring(L, 1); - auto tasks = const_cast(scope->get_tasks()); + const auto *task_name = luaL_checkstring(L, 1); + auto *tasks = const_cast(scope->get_tasks()); if (tasks->task_exists(task_name)) { - auto other = tasks->get_task(task_name); + const auto *other = tasks->get_task(task_name); return luaL_error( L, "Duplicate task definition with name <%s>, already defined at %s", @@ -258,7 +255,7 @@ namespace kafe::scripting { + string(":") + to_string(ar.currentline); - auto task = new Task( + auto *task = new Task( task_name, function_reference, source @@ -275,7 +272,7 @@ namespace kafe::scripting { } int lua_api_on_role_invoke(lua_State *L) { - auto scope = get_scope(L); + auto *scope = get_scope(L); auto *logger = const_cast(scope->get_context()->get_log_listener()); if (scope->has_current_api()) { @@ -307,7 +304,7 @@ namespace kafe::scripting { skip_empty = static_cast(lua_toboolean(L, 3)); } - auto role = luaL_checkstring(L, 1); + const auto *role = luaL_checkstring(L, 1); auto function_reference = luaL_ref(L, LUA_REGISTRYINDEX); auto inventory_items = scope->get_inventory()->find_for_scope( @@ -327,11 +324,12 @@ namespace kafe::scripting { logger->context_push(string(role)); bool failed = false; - for (auto item : inventory_items) { + for (const auto *item : inventory_items) { auto remote_id = item->remote_id(); logger->emit_info("Entering node <%s>", remote_id.c_str()); logger->context_push(remote_id); - auto ssh_manager = SshManager(scope->get_ssh_pool(), item); + const auto *envvals = scope->get_context()->get_envvals(); + auto ssh_manager = SshManager(scope->get_ssh_pool(), envvals, item); auto ssh_api = SshApi(&ssh_manager, logger); scope->set_current_remote(&ssh_api); @@ -343,8 +341,9 @@ namespace kafe::scripting { if (status) { failed = true; - auto lua_error = lua_tostring(L, -1); - logger->emit_warning("%s", lua_error); + const auto *lua_error = lua_tostring(L, -1); + const auto with_vars = scope->replace_vars(lua_error); + logger->emit_warning("%s", with_vars.c_str()); logger->context_pop(); break; } @@ -359,7 +358,7 @@ namespace kafe::scripting { } int lua_api_remote_within(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (!scope->has_current_api()) { return luaL_error(L, "Can not change remote directory when not in remote scope"); @@ -370,7 +369,7 @@ namespace kafe::scripting { } auto directory = scope->replace_vars(luaL_checkstring(L, 1)); - auto api = const_cast(scope->get_current_api()); + auto *api = const_cast(scope->get_current_api()); scope->get_context()->get_log_listener()->emit_info( "Changing remote working directory for current context to <%s>", @@ -383,7 +382,7 @@ namespace kafe::scripting { } int lua_api_remote_exec(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (!scope->has_current_api()) { return luaL_error(L, "Can not execute remote command when not in remote scope"); @@ -408,7 +407,7 @@ namespace kafe::scripting { } auto command = scope->replace_vars(luaL_checkstring(L, 1)); - auto api = scope->get_current_api(); + const auto *api = scope->get_current_api(); auto result = api->execute(command, print_output); @@ -420,7 +419,7 @@ namespace kafe::scripting { } int lua_api_remote_shell(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (!scope->has_current_api()) { return luaL_error(L, "Can not execute remote command when not in remote scope"); @@ -437,7 +436,7 @@ namespace kafe::scripting { } auto command = scope->replace_vars(luaL_checkstring(L, 1)); - auto api = scope->get_current_api(); + const auto *api = scope->get_current_api(); auto result = api->execute(command, true); @@ -447,10 +446,13 @@ namespace kafe::scripting { } int lua_api_archive_dir_tmp(lua_State *L) { - auto scope = get_scope(L); + auto *scope = get_scope(L); if (scope->has_current_api()) { - // TODO: issue warning + scope->get_context()->get_log_listener()->emit_warning( + "Creating archive while connected to remote server. Archive will be created per each" + "remote server. This is usually not optimal." + ); } if (1 != lua_gettop(L) || !lua_isstring(L, 1)) { @@ -481,10 +483,13 @@ namespace kafe::scripting { } int lua_api_archive_dir(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (scope->has_current_api()) { - // TODO: issue warning + scope->get_context()->get_log_listener()->emit_warning( + "Creating archive while connected to remote server. Archive will be created per each" + "remote server. This is usually not optimal." + ); } if (2 != lua_gettop(L) || !lua_isstring(L, 1) || !lua_isstring(L, 2)) { @@ -514,7 +519,7 @@ namespace kafe::scripting { } int lua_api_upload_file(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (!scope->has_current_api()) { return luaL_error(L, "Can not upload files when not in remote scope"); @@ -539,7 +544,7 @@ namespace kafe::scripting { auto local_file_norm = FileSystem::normalize(local_file, scope->get_local_api()->get_chdir()); - auto api = scope->get_current_api(); + const auto *api = scope->get_current_api(); auto timer = scope->get_context()->get_log_listener()->emit_info_wt( "Uploading local file <%s> to remote <%s>", @@ -569,7 +574,7 @@ namespace kafe::scripting { } int lua_api_download_file(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (!scope->has_current_api()) { return luaL_error(L, "Can not download files when not in remote scope"); @@ -594,7 +599,7 @@ namespace kafe::scripting { auto local_file_norm = FileSystem::normalize(local_file, scope->get_local_api()->get_chdir()); - auto api = scope->get_current_api(); + const auto *api = scope->get_current_api(); auto timer = scope->get_context()->get_log_listener()->emit_info_wt( "Downloading file <%s> from remote <%s>", @@ -624,7 +629,7 @@ namespace kafe::scripting { } int lua_api_upload_str(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (!scope->has_current_api()) { return luaL_error(L, "Can not upload files when not in remote scope"); @@ -647,7 +652,7 @@ namespace kafe::scripting { auto content = scope->replace_vars(luaL_checkstring(L, 1)); auto remote_file = scope->replace_vars(luaL_checkstring(L, 2)); - auto api = scope->get_current_api(); + const auto *api = scope->get_current_api(); auto timer = scope->get_context()->get_log_listener()->emit_info_wt( "Uploading string as file to remote <%s>", @@ -676,7 +681,7 @@ namespace kafe::scripting { } int lua_api_download_str(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (!scope->has_current_api()) { return luaL_error(L, "Can not download files when not in remote scope"); @@ -693,7 +698,7 @@ namespace kafe::scripting { } auto remote_file = scope->replace_vars(luaL_checkstring(L, 1)); - auto api = scope->get_current_api(); + const auto *api = scope->get_current_api(); auto timer = scope->get_context()->get_log_listener()->emit_info_wt( "Downloading string from remote <%s>", remote_file.c_str() @@ -720,7 +725,7 @@ namespace kafe::scripting { } int lua_api_define(lua_State *L) { - auto scope = get_scope(L); + auto *scope = get_scope(L); int n_args = lua_gettop(L); @@ -728,7 +733,7 @@ namespace kafe::scripting { return luaL_error(L, "Expected two arguments"); } - auto key = luaL_checkstring(L, 1); + const auto *key = luaL_checkstring(L, 1); auto value = scope->replace_vars(lua_tostring(L, 2)); scope->get_context()->get_log_listener()->emit_debug( @@ -743,14 +748,14 @@ namespace kafe::scripting { } int lua_api_strfvars(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); int n_args = lua_gettop(L); if (1 != n_args) { return luaL_error(L, "Expected one argument"); } - auto input = luaL_checkstring(L, 1); + const auto *input = luaL_checkstring(L, 1); try { lua_pushstring(L, scope->replace_vars(input).c_str()); @@ -762,21 +767,36 @@ namespace kafe::scripting { } int lua_api_strfenv(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); int n_args = lua_gettop(L); if (1 != n_args) { return luaL_error(L, "Expected one argument"); } - auto input = luaL_checkstring(L, 1); + const auto *input = luaL_checkstring(L, 1); lua_pushstring(L, scope->replace_env(input).c_str()); return 1; } + int lua_api_getenv(lua_State *L) { + const auto *scope = get_scope(L); + int n_args = lua_gettop(L); + + if (1 != n_args) { + return luaL_error(L, "Expected one argument"); + } + + const auto *input = luaL_checkstring(L, 1); + const string val = scope->get_env(input); + lua_pushstring(L, val.empty() ? nullptr : val.c_str()); + + return 1; + } + int lua_api_local_within(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (1 != lua_gettop(L) || !lua_isstring(L, 1)) { luaL_error(L, "Expected one argument - string"); @@ -800,7 +820,7 @@ namespace kafe::scripting { } int lua_api_local_exec(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (scope->has_current_api()) { auto debug = get_lua_debug(L); @@ -840,7 +860,7 @@ namespace kafe::scripting { } int lua_api_local_shell(lua_State *L) { - auto scope = get_scope(L); + const auto *scope = get_scope(L); if (scope->has_current_api()) { auto debug = get_lua_debug(L); @@ -887,6 +907,7 @@ namespace kafe::scripting { {"define", lua_api_define}, {"strfvars", lua_api_strfvars}, {"strfenv", lua_api_strfenv}, + {"getenv", lua_api_getenv}, {"local_exec", lua_api_local_exec}, {"local_shell", lua_api_local_shell}, {"local_within", lua_api_local_within}, @@ -894,7 +915,8 @@ namespace kafe::scripting { }; extern "C" int lua_module_init(lua_State *L) { -#if LUA_VERSION_NUM > 501 + const auto *scope = get_scope(L); + lua_newtable(L); luaL_setfuncs(L, module_def, 0); @@ -913,16 +935,38 @@ namespace kafe::scripting { lua_pushinteger(L, LIBKAFE_API_LEVEL); lua_setfield(L, -2, "api_level"); + lua_pushstring(L, scope->get_context()->get_environment().c_str()); + lua_setfield(L, -2, "environment"); + lua_pushvalue(L, -1); -#else - luaL_register(L, LIBKAFE_LUA_MODULE_NAME, module_def); -#endif + return 1; } extern "C" void lua_bootstrap(lua_State *L) { + const auto *scope = get_scope(L); + auto extra_args = scope->get_extra_args(); + + // Init "kafe" module luaL_requiref(L, LIBKAFE_LUA_MODULE_NAME, lua_module_init, false); lua_pop(L, 1); + + // Populate "arg" + lua_createtable(L, extra_args.size() + 1, 0); + int arg_table = lua_gettop(L); + + //// Push project file name + lua_pushstring(L, scope->get_project_file().c_str()); + lua_rawseti(L, arg_table, 0); + + //// Push remaining args + long long index = 1; + for (auto arg: extra_args) { + lua_pushstring(L, arg.c_str()); + lua_rawseti(L, arg_table, index); + ++index; + } + lua_setglobal(L, "arg"); } // END integration with Lua