Python-pyo3: add library interface (#566)

* cargo update * rename p_vault to n_vault again and move cli to cli.py * expose list_all and lookup from rust library * output binary data in base64 * rust fixes * add license texts * implement store, exists, and delete * update pyproject information * add integration test cases for python library * tweak comment * fix integration test trigger for PRs * cancel previous runs for PR wheel build * fix store when input argument is a string * fix empty string for print end arg * add extra rust cache key for release build * convert lookup bytes output to string * workaround list all case since there seems to be unexpected extra keys * remove unneeded decode from python rust lib test * add `delete_many` to library * fix wc -l to not count empty lines * remove extra list all * add readme section for python library usage * support specifying vault parameters * add init and update * add stack status * return python dict * update docstrings for library functions * docstrings for all public library methods * match parameter names to old python library * fix variable rename * make type hinting compatible with python 3.9 * fix help output when no arguments are given * rename and reorder vault class parameters to match previous version * add direct encrypt and decrypt * return bytes from direct encrypt and decrypt * fix type hints for 3.9 :( * ignore all venvs * match parameter name for direct decrypt * cargo update * fix typo in helper function name
NitorCreations · Nov 14, 2024 · 6586709 · 6586709
1 parent 0466f02
commit 6586709
Show file tree

Hide file tree

Showing 21 changed files with 837 additions and 78 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -8,8 +8,6 @@ on:
     paths:
       - "!**/README.md"
   pull_request:
-    paths:
-      - "!**/README.md"
 
 permissions:
   id-token: write
@@ -37,6 +35,9 @@ jobs:
 
       - uses: Swatinem/[email protected]
         if: ${{ matrix.lang == 'rust'}}
+        with:
+          # The build script creates a `release` build so use separate cache
+          key: "release"
 
       - uses: actions/setup-go@v5
         if: ${{ matrix.lang == 'go'}}
@@ -218,9 +219,6 @@ jobs:
       - name: Delete secret with Rust
         run: bin/rust/vault -d "secret-${{github.sha}}.zip"
 
-      - name: Verify that key has been deleted with Rust
-        run: bin/rust/vault exists secret-${{github.sha}}.zip | grep -q "does not exist"
-
       - name: Verify that keys have been deleted using Rust
         run: |
           bin/rust/vault exists secret-python | grep -q "key 'secret-python' does not exist"
@@ -229,6 +227,33 @@ jobs:
           bin/rust/vault exists secret-rust | grep -q "key 'secret-rust' does not exist"
           bin/rust/vault exists secret-nodejs | grep -q "key 'secret-nodejs' does not exist"
 
+      - name: Check Python vault package
+        run: python -m pip show nitor-vault
+
+      - name: Store secret using Python library
+        run: |
+          python -c "from n_vault import Vault; Vault().store('secret-python-library', 'sha-${{github.sha}}')"
+
+      - name: Verify secret using Python library
+        run: |
+          python -c "from n_vault import Vault; print('true') if Vault().exists('secret-python-library') else print('false')" | grep -q "true"
+
+      - name: Validate storing worked with Rust
+        run: diff <(bin/rust/vault -l secret-python-library) <(echo -n sha-${{github.sha}})
+
+      - name: Lookup with Python library
+        run: |
+          diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library').decode('utf-8'), end='', flush=True)") <(echo -n sha-${{github.sha}})
+
+      - name: List with Python library
+        run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))"
+
+      - name: Delete with Python library
+        run: python -c "from n_vault import Vault; Vault().delete('secret-python-library')"
+
+      - name: Verify that key has been deleted with Rust
+        run: bin/rust/vault exists secret-python-library | grep -q "key 'secret-python-library' does not exist"
+
       - name: Install Python PyO3 vault
         run: python -m pip install --force-reinstall .
         working-directory: python-pyo3
@@ -310,3 +335,36 @@ jobs:
 
       - name: Verify that key has been deleted with Python-pyo3
         run: vault exists secret-${{github.sha}}.zip | grep -q "does not exist"
+
+      - name: Check Python vault package
+        run: python -m pip show nitor-vault
+
+      - name: Delete all keys with Python library
+        run: python -c "from n_vault import Vault; Vault().delete_many(Vault().list_all())"
+
+      - name: List with Python library
+        run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | grep -ve '^\s*$' | wc -l | grep -q "0"
+
+      - name: Store secret using Python library
+        run: |
+          python -c "from n_vault import Vault; Vault().store('secret-python-library', 'sha-${{github.sha}}')"
+
+      - name: Verify secret using Python library
+        run: |
+          python -c "from n_vault import Vault; print('true') if Vault().exists('secret-python-library') else print('false')" | grep -q "true"
+
+      - name: Validate storing worked with Rust
+        run: diff <(bin/rust/vault -l secret-python-library) <(echo -n sha-${{github.sha}})
+
+      - name: Lookup with Python library
+        run: |
+          diff <(python -c "from n_vault import Vault; print(Vault().lookup('secret-python-library'), end='', flush=True)") <(echo -n sha-${{github.sha}})
+
+      - name: List with Python library
+        run: python -c "from n_vault import Vault; print('\n'.join(Vault().list_all()))" | wc -l | grep -q "1"
+
+      - name: Delete with Python library
+        run: python -c "from n_vault import Vault; Vault().delete('secret-python-library')"
+
+      - name: Verify that key has been deleted with Rust
+        run: bin/rust/vault exists secret-python-library | grep -q "key 'secret-python-library' does not exist"
diff --git a/.github/workflows/python-wheel.yml b/.github/workflows/python-wheel.yml
@@ -28,6 +28,11 @@ on:
 permissions:
   contents: read
 
+# Cancel previous runs for PRs but not pushes to main
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+  cancel-in-progress: true
+
 jobs:
   linux:
     runs-on: ${{ matrix.platform.runner }}

diff --git a/.gitignore b/.gitignore
@@ -79,7 +79,8 @@ celerybeat-schedule
 .env
 
 # virtualenv
-venv/
+venv*/
+.venv*/
 ENV/
 
 # Spyder project settings
@@ -112,4 +113,3 @@ dependency-reduced-pom.xml
 # Go binary
 /go/vault
 /go/vault
-.venv
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -5,7 +5,7 @@ resolver = "2"
 
 [workspace.package]
 edition = "2021"
-version = "2.1.2"
+version = "2.2.0"
 
 [profile.release]
 lto = "thin"
diff --git a/python-pyo3/README.md b/python-pyo3/README.md
@@ -4,7 +4,7 @@ Python vault implementation using the Rust vault library.
 
 See the [repo](https://github.com/NitorCreations/vault) root readme for more general information.
 
-## Usage
+## Vault CLI
 
 ```console
 Encrypted AWS key-value storage utility
@@ -40,12 +40,13 @@ Options:
   -V, --version            Print version
 ```
 
-## Install
+### Install
 
-### From PyPI
+#### From PyPI
 
 Use [pipx](https://github.com/pypa/pipx) or [uv](https://github.com/astral-sh/uv)
-to install the Python vault package from [PyPI](https://pypi.org/project/nitor-vault/) globally in an isolated environment.
+to install the Python vault package from [PyPI](https://pypi.org/project/nitor-vault/)
+globally in an isolated environment.
 
 ```shell
 pipx install nitor-vault
@@ -55,7 +56,7 @@ uv tool install nitor-vault
 
 The command `vault` should now be available in path.
 
-### From source
+#### From source
 
 Build and install locally from source code using pip.
 This requires a [Rust toolchain](https://rustup.rs/) to be able to build the Rust library.
@@ -77,6 +78,33 @@ and will not be available in path globally.
 which -a vault
 ```
 
+## Vault library
+
+This Python package can also be used as a Python library to interact with the Vault directly from Python code.
+
+Add the `nitor-vault` package to your project dependencies,
+or install directly with pip.
+
+Example usage:
+
+```python
+from n_vault import Vault
+
+if not Vault().exists("key"):
+    Vault().store("key", "value")
+
+keys = Vault().list_all()
+
+value = Vault().lookup("key")
+
+if Vault().exists("key"):
+    Vault().delete("key")
+
+# specify vault parameters
+vault = Vault(vault_stack="stack-name", profile="aws-credentials-name")
+value = vault.lookup("key")
+```
+
 ## Development
 
 Uses:
@@ -112,9 +140,9 @@ Run Python CLI:
 
 ```shell
 # uv
-uv run python/p_vault/vault.py -h
+uv run python/n_vault/cli.py -h
 # venv
-python3 python/p_vault/vault.py -h
+python3 python/n_vault/cli.py -h
 ```
 
 Install and run vault inside virtual env:

diff --git a/python-pyo3/pyproject.toml b/python-pyo3/pyproject.toml
@@ -22,22 +22,23 @@ authors = [
     { name = "Pasi Niemi", email = "[email protected]" },
     { name = "Akseli Lukkarila", email = "[email protected]" },
 ]
-license = { text = "Apache 2.0" }
+license = { text = "Apache-2.0" }
 classifiers = [
     "Programming Language :: Rust",
     "Programming Language :: Python :: Implementation :: CPython",
 ]
 dependencies = []
 
 [project.optional-dependencies]
-build = ["maturin", "twine", "wheel"]
-dev = ["ruff"]
+build = ["maturin", "wheel"]
+dev = ["maturin", "ruff"]
 
 [project.urls]
 Repository = "https://github.com/NitorCreations/vault"
+Homepage = "https://github.com/NitorCreations/vault"
 
 [project.scripts]
-vault = "p_vault.vault:main"
+vault = "n_vault.cli:main"
 
 [build-system]
 requires = ["maturin>=1.7,<2.0"]
@@ -46,9 +47,9 @@ build-backend = "maturin"
 [tool.maturin]
 bindings = "pyo3"
 features = ["pyo3/extension-module"]
-module-name = "p_vault.nitor_vault_rs"
+module-name = "n_vault.nitor_vault_rs"
 profile = "release"
-python-packages = ["p_vault"]
+python-packages = ["n_vault"]
 python-source = "python"
 strip = true
 
@@ -59,7 +60,7 @@ venv = ".venv"
 [tool.ruff]
 # https://docs.astral.sh/ruff/configuration/
 include = ["*.py", "*.pyi", "**/pyproject.toml"]
-target-version = "py311"
+target-version = "py39"
 line-length = 120
 
 [tool.ruff.lint]