Merge pull request #1119 from OpenFn/audit_trail

for #271, set up audit_events

CHANGELOG.md

@@ -12,6 +12,9 @@ and this project adheres to
 
 ### Changed
 
+- Modified audit trail to handle lots of different kind of audit events
+  [#271](https://github.com/OpenFn/Lightning/issues/271)/[#44](https://github.com/OpenFn/Lightning/issues/44)
+
 ### Fixed
 
 - Fix randomly unresponsive job panel after job deletion

lib/lightning/auditing.ex

@@ -8,7 +8,9 @@ defmodule Lightning.Auditing do
 
   def list_all(params \\ %{}) do
     from(a in Lightning.Credentials.Audit,
-      preload: [:actor],
+      left_join: u in Lightning.Accounts.User,
+      on: [id: a.actor_id],
+      select_merge: %{actor: u},
       order_by: [desc: a.inserted_at]
     )
     |> Repo.paginate(params)

lib/lightning/auditing/model.ex

@@ -8,6 +8,7 @@ defmodule Lightning.Auditing.Model do
   defmacro __using__(opts) do
     repo = Keyword.fetch!(opts, :repo)
     schema = Keyword.fetch!(opts, :schema)
+    item = Keyword.fetch!(opts, :item)
     events = Keyword.fetch!(opts, :events)
 
     if Enum.empty?(events),
@@ -27,24 +28,25 @@ defmodule Lightning.Auditing.Model do
 
     event_signature =
       quote do
-        def event(event, row_id, actor_id, metadata \\ %{})
+        def event(event, item_id, actor_id, changes \\ %{})
       end
 
     event_log_functions =
       for event_name <- events do
         quote do
           # Output:
           #
-          # def event("foo_event", row_id, actor_id, metadata) do
-          #   Lightning.Audit.event(schema, "foo_event", row_id, actor_id, metadata)
+          # def event(item_type, "foo_event", item_id, actor_id, changes) do
+          #   Lightning.Audit.event(item_type, schema, "foo_event", item_id, actor_id, changes)
           # end
-          def event(unquote(event_name), row_id, actor_id, metadata) do
+          def event(unquote(event_name), item_id, actor_id, changes) do
             unquote(__MODULE__).event(
               unquote(schema),
+              unquote(item),
               unquote(event_name),
-              row_id,
+              item_id,
               actor_id,
-              metadata
+              changes
             )
           end
         end
@@ -78,34 +80,36 @@ defmodule Lightning.Auditing.Model do
   end
 
   @doc """
-  Creates a `schema` changeset for the `event` identified by `row_id` and caused
+  Creates a `schema` changeset for the `event` identified by `item_id` and caused
   by `actor_id`.
 
-  The given `metadata` can be either `nil`, `Ecto.Changeset`, struct or map.
+  The given `changes` can be either `nil`, `Ecto.Changeset`, struct or map.
 
-  It returns `:no_changes` in case of an `Ecto.Changeset` metadata that changed nothing
+  It returns `:no_changes` in case of an `Ecto.Changeset` changes that changed nothing
   or an `Ecto.Changeset` with the event ready to be inserted.
   """
   @spec event(
           module(),
           String.t(),
+          String.t(),
           Ecto.UUID.t(),
           Ecto.UUID.t(),
           Ecto.Changeset.t() | map() | nil
         ) ::
           :no_changes | Ecto.Changeset.t()
 
-  def event(schema, event, row_id, actor_id, metadata \\ %{})
+  def event(schema, item_type, event, item_id, actor_id, changes \\ %{})
 
-  def event(_, _, _, _, %Ecto.Changeset{changes: changes} = _changeset)
+  def event(_, _, _, _, _, %Ecto.Changeset{changes: changes} = _changeset)
       when map_size(changes) == 0 do
     :no_changes
   end
 
   def event(
         schema,
+        item_type,
         event,
-        row_id,
+        item_id,
         actor_id,
         %Ecto.Changeset{data: %subject_schema{} = data, changes: changes}
       ) do
@@ -117,26 +121,28 @@ defmodule Lightning.Auditing.Model do
       |> MapSet.intersection(change_keys)
       |> MapSet.to_list()
 
-    metadata = %{
+    changes = %{
       before: Map.take(data, field_keys),
       after: Map.take(changes, field_keys)
     }
 
-    audit_changeset(schema, event, row_id, actor_id, metadata)
+    audit_changeset(schema, item_type, event, item_id, actor_id, changes)
   end
 
-  def event(schema, event, row_id, actor_id, metadata) when is_map(metadata) do
-    audit_changeset(schema, event, row_id, actor_id, metadata)
+  def event(schema, item_type, event, item_id, actor_id, changes)
+      when is_map(changes) do
+    audit_changeset(schema, item_type, event, item_id, actor_id, changes)
   end
 
-  defp audit_changeset(schema, event, row_id, actor_id, metadata) do
+  defp audit_changeset(schema, item_type, event, item_id, actor_id, changes) do
     schema
     |> struct()
     |> schema.changeset(%{
+      item_type: item_type,
       event: event,
-      row_id: row_id,
+      item_id: item_id,
       actor_id: actor_id,
-      metadata: metadata
+      changes: changes
     })
   end
 end

lib/lightning/credentials.ex

@@ -181,7 +181,12 @@ defmodule Lightning.Credentials do
         |> Multi.insert(
           :audit,
           fn %{credential: credential} ->
-            Audit.event("updated", credential.id, credential.user_id, changeset)
+            Audit.event(
+              "updated",
+              credential.id,
+              credential.user_id,
+              changeset
+            )
           end
         )
         |> Multi.append(project_credentials_multi)
@@ -199,13 +204,17 @@ defmodule Lightning.Credentials do
       multi,
       {:audit, Ecto.Changeset.get_field(changeset, :project_id)},
       fn %{credential: credential} ->
-        "removed_from_project"
-        |> Audit.event(credential.id, credential.user_id, %{
-          before: %{
-            project_id: Ecto.Changeset.get_field(changeset, :project_id)
-          },
-          after: %{project_id: nil}
-        })
+        Audit.event(
+          "removed_from_project",
+          credential.id,
+          credential.user_id,
+          %{
+            before: %{
+              project_id: Ecto.Changeset.get_field(changeset, :project_id)
+            },
+            after: %{project_id: nil}
+          }
+        )
       end
     )
   end
@@ -221,8 +230,7 @@ defmodule Lightning.Credentials do
       multi,
       {:audit, Ecto.Changeset.get_field(changeset, :project_id)},
       fn %{credential: credential} ->
-        "added_to_project"
-        |> Audit.event(credential.id, credential.user_id, %{
+        Audit.event("added_to_project", credential.id, credential.user_id, %{
           before: %{project_id: nil},
           after: %{
             project_id: Ecto.Changeset.get_field(changeset, :project_id)

lib/lightning/credentials/audit.ex

@@ -5,6 +5,7 @@ defmodule Lightning.Credentials.Audit do
   use Lightning.Auditing.Model,
     repo: Lightning.Repo,
     schema: __MODULE__,
+    item: "credential",
     events: [
       "created",
       "updated",
@@ -13,7 +14,7 @@ defmodule Lightning.Credentials.Audit do
       "deleted"
     ]
 
-  defmodule Metadata do
+  defmodule Changes do
     @moduledoc false
 
     use Ecto.Schema
@@ -26,8 +27,8 @@ defmodule Lightning.Credentials.Audit do
     end
 
     @doc false
-    def changeset(metadata, attrs \\ %{}) do
-      metadata
+    def changeset(changes, attrs \\ %{}) do
+      changes
       |> cast(attrs, [:before, :after])
       |> update_change(:before, &encrypt_body/1)
       |> update_change(:after, &encrypt_body/1)
@@ -49,24 +50,24 @@ defmodule Lightning.Credentials.Audit do
   use Ecto.Schema
   import Ecto.Changeset
 
-  alias Lightning.Accounts.User
-
   @primary_key {:id, :binary_id, autogenerate: true}
   @foreign_key_type :binary_id
-  schema "credentials_audit" do
+  schema "audit_events" do
     field :event, :string
-    field :row_id, Ecto.UUID
-    embeds_one :metadata, Metadata
-    belongs_to :actor, User
+    field :item_type, :string
+    field :item_id, Ecto.UUID
+    embeds_one :changes, Changes
+    field :actor_id, Ecto.UUID
+    field :actor, :map, virtual: true
 
     timestamps(updated_at: false)
   end
 
   @doc false
   def changeset(%__MODULE__{} = audit, attrs) do
     audit
-    |> cast(attrs, [:event, :row_id, :actor_id])
-    |> cast_embed(:metadata)
+    |> cast(attrs, [:event, :item_id, :actor_id, :item_type])
+    |> cast_embed(:changes)
     |> validate_required([:event, :actor_id])
   end
 end

lib/lightning_web/live/audit_live/index.html.heex

@@ -8,36 +8,47 @@
     <.table>
       <.tr>
         <.th>Occurred</.th>
-        <.th class="text-center">Event</.th>
+        <.th>Event</.th>
         <.th>Actor</.th>
         <.th>Subject</.th>
       </.tr>
-
       <%= for audit <- @page.entries do %>
         <.tr id={"audit-#{audit.id}"} class="border-dotted border-gray-100">
           <.td>
             <%= audit.inserted_at |> Calendar.strftime("%c %Z") %>
           </.td>
-          <.td class="text-center">
+          <.td>
             <.badge color="success" label={audit.event} />
           </.td>
-          <.td><%= audit.actor.email %></.td>
           <.td>
             <div class="flex flex-col overflow-hidden">
               <div class="overflow-hidden font-normal text-gray-900 whitespace-nowrap text-ellipsis dark:text-gray-300">
-                Credential
+                <%= if audit.actor,
+                  do: "#{audit.actor.first_name} #{audit.actor.last_name}",
+                  else: "(User deleted)" %>
+              </div>
+              <div class="overflow-hidden font-normal text-gray-500 text-xs whitespace-nowrap text-ellipsis">
+                <%= if audit.actor,
+                  do: audit.actor.email,
+                  else: display_short_uuid(audit.actor_id) %>
+              </div>
+            </div>
+          </.td>
+          <.td>
+            <div class="flex flex-col overflow-hidden">
+              <div class="overflow-hidden font-normal text-gray-900 whitespace-nowrap text-ellipsis dark:text-gray-300">
+                <%= audit.item_type |> String.split(".") |> Enum.at(-1) %>
               </div>
               <div class="overflow-hidden font-normal text-gray-500 text-xs whitespace-nowrap text-ellipsis">
-                <%= display_short_uuid(audit.row_id) %>
+                <%= display_short_uuid(audit.item_id) %>
               </div>
             </div>
           </.td>
         </.tr>
-
         <.tr>
-          <%= if audit.metadata.after do %>
+          <%= if audit.changes.after do %>
             <.td colspan="4" class="font-mono text-xs break-all">
-              <.diff metadata={audit.metadata} />
+              <.diff metadata={audit.changes} />
             </.td>
           <% else %>
             <.td colspan="4" class="font-mono text-xs">

priv/repo/migrations/20230916052400_change_credentials_audit_to_audit_events.exs

@@ -0,0 +1,41 @@
+defmodule Lightning.Repo.Migrations.ChangeCredentialsAuditToAuditEvents do
+  use Ecto.Migration
+
+  def up do
+    rename_index(from: "credentials_audit_pkey", to: "audit_events_pkey")
+    rename_index(from: "credentials_audit_row_id_index", to: "audit_events_item_id_index")
+    drop constraint("credentials_audit", "credentials_audit_actor_id_fkey")
+
+    rename table("credentials_audit"), :row_id, to: :item_id
+    rename table("credentials_audit"), :metadata, to: :changes
+    create index("credentials_audit", [:actor_id])
+
+    rename table(:credentials_audit), to: table(:audit_events)
+
+    alter table("audit_events") do
+      add :item_type, :string, default: "credential"
+    end
+  end
+
+  defp rename_constraint(table, from: from, to: to) do
+    execute(
+      """
+      ALTER TABLE #{table} RENAME CONSTRAINT "#{from}" TO "#{to}";
+      """,
+      """
+      ALTER TABLE #{table} RENAME CONSTRAINT "#{to}" TO "#{from}";
+      """
+    )
+  end
+
+  defp rename_index(from: from, to: to) do
+    execute(
+      """
+      ALTER INDEX #{from} RENAME TO #{to};
+      """,
+      """
+      ALTER INDEX #{to} RENAME TO #{from};
+      """
+    )
+  end
+end

priv/repo/migrations/20230916055521_remove_defauilt_item_type_for_audit_events.exs

@@ -0,0 +1,9 @@
+defmodule Lightning.Repo.Migrations.RemoveDefauiltItemTypeForAuditEvents do
+  use Ecto.Migration
+
+  def change do
+    alter table(:audit_events) do
+      modify(:item_type, :string, default: nil, null: false)
+    end
+  end
+end

test/lightning/auditing_test.exs

@@ -10,7 +10,7 @@ defmodule Lightning.AuditingTest do
 
       %{entries: [entry]} = Auditing.list_all()
 
-      assert entry.row_id == credential_id
+      assert entry.item_id == credential_id
     end
   end
 end