Merge pull request #2 from acrlabs/drmorr/display-diff

Drmorr/display diff

.coveragerc

@@ -0,0 +1,22 @@
+[run]
+branch = True
+source =
+    fireconfig
+omit =
+    fireconfig/k8s/*
+
+[report]
+exclude_lines =
+    # Have to re-enable the standard pragma
+    \#\s*pragma: no cover
+
+    # Don't complain if tests don't hit defensive assertion code:
+    ^\s*raise AssertionError\b
+    ^\s*raise NotImplementedError\b
+    ^\s*return NotImplemented\b
+    ^\s*raise$
+
+    # Don't complain if non-runnable code isn't run:
+    ^if __name__ == ['"]__main__['"]:$
+
+# vim:ft=dosini

.flake8

@@ -1,3 +1,6 @@
 [flake8]
 max-line-length = 121
-ignore = E121,E123,E126,E226,E24,E704,W503,W504,E702,E703,E741,W605
+extend-ignore = E702,E703,E741,W605,E124,E128
+extend-exclude = fireconfig/k8s/*
+
+# vim:ft=dosini

.github/workflows/test.yml

@@ -0,0 +1,25 @@
+name: Run tests
+
+on:
+  push:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out master
+        uses: actions/checkout@v4
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Run tests
+        run: |
+          poetry install
+          make test

.gitignore

@@ -1,2 +1,3 @@
 __pycache__
 .*sw[op]
+.coverage

.pre-commit-config.yaml

@@ -6,11 +6,14 @@ repos:
     hooks:
       - id: end-of-file-fixer
       - id: check-yaml
+        args: ["--allow-multiple-documents"]
       - id: trailing-whitespace
-  - repo: https://github.com/asottile/reorder-python-imports
-    rev: v3.10.0
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
     hooks:
-      - id: reorder-python-imports
+      - id: isort
+        args:
+          - --sl
   - repo: https://github.com/PyCQA/flake8
     rev: 6.0.0
     hooks:
@@ -19,4 +22,4 @@ repos:
     rev: v1.4.1
     hooks:
       - id: mypy
-        additional_dependencies: [cdk8s]
+        additional_dependencies: [cdk8s, types-simplejson, types-pyyaml]

Makefile

@@ -0,0 +1,6 @@
+.PHONY: test
+
+test:
+	poetry run coverage erase
+	poetry run coverage run -m pytest -svv itests
+	poetry run coverage report --show-missing

examples/README.md

@@ -0,0 +1,12 @@
+# 🔥Config Examples
+
+## Workflows
+
+The workflows directory contains a set of GitHub actions that you can use to have 🔥Config automatically compute the
+mermaid DAG and diff of changes to your Kubernetes objects, and then leave a comment on the PR with the DAG and diff.
+You _should_ just be able to copy these into your `.github/workflows` directory.  You'll need to set up a personal
+access token (PAT) with read access to your actions and read and write access to pull requests.  This PAT then needs to
+be injected into your actions as a GitHub secret.
+
+- [Managing your Personal Access Tokens](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens)
+- [Using secrets in GitHub Actions](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions)

examples/workflows/k8s_plan.yml

@@ -0,0 +1,49 @@
+name: Compute k8s plan
+
+on:
+  pull_request:
+    paths:
+      - 'k8s/**'
+
+jobs:
+  plan:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out master
+        uses: actions/checkout@v4
+        with:
+          ref: master
+          submodules: recursive
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Compile k8s charts
+        run: make k8s
+
+      - name: Check out PR
+        uses: actions/checkout@v4
+        with:
+          clean: false
+
+      - name: Compute dag/diff
+        run: make k8s
+
+      - name: Save artifacts
+        run: |
+          mkdir -p ./artifacts
+          echo ${{ github.event.number }} > ./artifacts/PR
+          mv .build/dag.mermaid ./artifacts/dag.mermaid
+          mv .build/k8s.df ./artifacts/k8s.df
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: k8s-plan-artifacts
+          path: artifacts/

examples/workflows/pr_comment_finished.yml

@@ -0,0 +1,58 @@
+name: Comment on the PR
+
+on:
+  workflow_run:
+    workflows: ["Compute k8s plan"]
+    types:
+      - completed
+
+jobs:
+  pr-comment:
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success'
+
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: k8s-plan-artifacts
+          github-token: ${{ secrets.PR_COMMENT_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          path: k8s-plan-artifacts
+
+      - name: Get PR number
+        uses: mathiasvr/[email protected]
+        id: pr
+        with:
+          run: cat k8s-plan-artifacts/PR
+
+      - name: Find previous comment ID
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          token: ${{ secrets.PR_COMMENT_TOKEN }}
+          issue-number: ${{ steps.pr.outputs.stdout }}
+          body-includes: "<!-- 🔥config summary -->"
+
+      - name: Render Comment Template
+        run: |
+          echo "<!-- 🔥config summary -->" > fireconfig-comment.md
+          echo "## Kubernetes Object DAG" >> fireconfig-comment.md
+          cat k8s-plan-artifacts/dag.mermaid >> fireconfig-comment.md
+          echo '<img src="https://raw.githubusercontent.com/acrlabs/fireconfig/master/assets/new.png" width=10/> New object' >> fireconfig-comment.md
+          echo '<img src="https://raw.githubusercontent.com/acrlabs/fireconfig/master/assets/removed.png" width=10/> Deleted object' >> fireconfig-comment.md
+          echo '<img src="https://raw.githubusercontent.com/acrlabs/fireconfig/master/assets/changed.png" width=10/> Updated object' >> fireconfig-comment.md
+          echo '<img src="https://raw.githubusercontent.com/acrlabs/fireconfig/master/assets/pod_recreate.png" width=10/> Updated object (causes pod recreation)' >> fireconfig-comment.md
+          echo "## Detailed Diff" >> fireconfig-comment.md
+          cat k8s-plan-artifacts/k8s.df >> fireconfig-comment.md
+
+      - name: Comment on PR
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          token: ${{ secrets.PR_COMMENT_TOKEN }}
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ steps.pr.outputs.stdout }}
+          body-path: fireconfig-comment.md
+          edit-mode: replace

examples/workflows/pr_comment_starting.yml

@@ -0,0 +1,44 @@
+name: Update the PR Comment
+
+on:
+  #######################################################################################
+  # WARNING: DO NOT CHANGE THIS ACTION TO CHECK OUT OR EXECUTE ANY CODE!!!!!            #
+  #                                                                                     #
+  # This can allow an attacker to gain write access to code in the repository or read   #
+  # any repository secrets!  This should _only_ be used to update or add a PR comment.  #
+  #                                                                                     #
+  # See https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ #
+  # for more details.                                                                   #
+  #######################################################################################
+  pull_request_target:
+    paths:
+      - 'k8s/**'
+
+jobs:
+  pr-comment:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Find previous comment ID
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          token: ${{ secrets.PR_COMMENT_TOKEN }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body-includes: "<!-- 🔥config summary -->"
+
+      - name: Render Comment Template
+        run: |
+          echo
+
+      - name: Comment on PR
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          token: ${{ secrets.PR_COMMENT_TOKEN }}
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body: |
+            <!-- 🔥config summary -->
+            ## Updating Kubernetes DAG...
+            Please wait until the job has finished.
+          edit-mode: replace

fireconfig/__init__.py

@@ -1,11 +1,117 @@
-from .container import ContainerBuilder
-from .deployment import DeploymentBuilder
-from .env import EnvBuilder
-from .volume import VolumesBuilder
+import typing as T
+from abc import ABCMeta
+from abc import abstractmethod
+from collections import defaultdict
+
+from cdk8s import App
+from cdk8s import Chart
+from cdk8s import DependencyGraph
+from constructs import Construct
+
+from fireconfig.container import ContainerBuilder
+from fireconfig.deployment import DeploymentBuilder
+from fireconfig.env import EnvBuilder
+from fireconfig.namespace import add_missing_namespace
+from fireconfig.output import format_diff
+from fireconfig.output import format_mermaid_graph
+from fireconfig.plan import GLOBAL_CHART_NAME
+from fireconfig.plan import compute_diff
+from fireconfig.plan import find_deleted_nodes
+from fireconfig.plan import get_resource_changes
+from fireconfig.plan import walk_dep_graph
+from fireconfig.subgraph import ChartSubgraph
+from fireconfig.util import fix_cluster_scoped_objects
+from fireconfig.volume import VolumesBuilder
 
 __all__ = [
     'ContainerBuilder',
     'DeploymentBuilder',
     'EnvBuilder',
     'VolumesBuilder',
 ]
+
+
+class AppPackage(metaclass=ABCMeta):
+    """
+    Users should implement the AppPackage class to pass into fireconfig
+    """
+
+    @property
+    @abstractmethod
+    def id(self):
+        ...
+
+    @abstractmethod
+    def compile(self, app: Construct):
+        ...
+
+
+def compile(
+    pkgs: T.Dict[str, T.List[AppPackage]],
+    dag_filename: T.Optional[str] = None,
+    cdk8s_outdir: T.Optional[str] = None,
+    dry_run: bool = False,
+) -> T.Tuple[str, str]:
+    """
+    `compile` takes a list of "packages" and generates Kubernetes manifests from them.  It
+    also generates a Markdown-ified "diff" and a mermaid graph representing the Kubernetes
+    manifest structure and changes.
+
+    :param pkgs: the list of packages to compile
+    :param dag_filename: the location of a previous DAG, for use in generating diffs
+    :param cdk8s_outdir: where to save the generated Kubernetes manifests
+    :param dry_run: actually generate the manifests, or not
+
+    :returns: the mermaid DAG and markdown-ified diff as a tuple of strings
+    """
+
+    app = App(outdir=cdk8s_outdir)
+
+    # Anything that is a "global" dependency (e.g., namespaces) that should be generated before
+    # everything else, or that should only be generated once, belongs in the global chart
+    gl = Chart(app, GLOBAL_CHART_NAME, disable_resource_name_hashes=True)
+
+    # For each cdk8s chart, we generate a sub-DAG (stored in `subgraphs`) and then we connect
+    # all the subgraphs together via the `subgraph_dag`
+    subgraph_dag = defaultdict(list)
+    subgraphs = {}
+    subgraphs[GLOBAL_CHART_NAME] = ChartSubgraph(GLOBAL_CHART_NAME)
+
+    for ns, pkglist in pkgs.items():
+        add_missing_namespace(gl, ns)
+        for pkg in pkglist:
+            chart = Chart(app, pkg.id, namespace=ns, disable_resource_name_hashes=True)
+            chart.add_dependency(gl)
+            pkg.compile(chart)
+
+            fix_cluster_scoped_objects(chart)
+            subgraphs[pkg.id] = ChartSubgraph(pkg.id)
+            subgraph_dag[gl.node.id].append(pkg.id)
+
+    # cdk8s doesn't compute the full dependency graph until you call `synth`, and there's no
+    # public access to it at that point, which is annoying.  Until that point, the dependency
+    # graph only includes the dependencies that you've explicitly added.  The format is
+    #
+    # [root (empty node)] ---> leaf nodes of created objects ---> tree in reverse
+    #   |
+    #   -----> [list of chart objects]
+    #
+    # The consequence being that we need to start at the root node, walk forwards, look at all the things
+    # that have "chart" fields, and then from there walk in reverse.  It's somewhat annoying.
+    for obj in DependencyGraph(app.node).root.outbound:
+        walk_dep_graph(obj, subgraphs)
+    diff, kinds = compute_diff(app)
+    resource_changes = get_resource_changes(diff, kinds)
+
+    try:
+        find_deleted_nodes(subgraphs, resource_changes, dag_filename)
+    except Exception as e:
+        print(f"WARNING: {e}\nCould not read old DAG file, graph may be missing deleted nodes")
+
+    graph_str = format_mermaid_graph(subgraph_dag, subgraphs, dag_filename, resource_changes)
+    diff_str = format_diff(resource_changes)
+
+    if not dry_run:
+        app.synth()
+
+    return graph_str, diff_str