Export History (#2397)

Added an export feature that allows project collaborators to export work order history from the history page. 

This feature enables users to download a zipped file containing all work orders, runs, steps, logs, and I/O data clips based on current filtering criteria. 
The export is processed in the background, uploaded to GCP storage, and a signed URL is sent via email. 

Download links expire after 24 hours, and users can view and download exports from the project files section under the Settings.

---------

Co-authored-by: Stuart Corbishley <[email protected]>
Co-authored-by: Frank Midigo <[email protected]>
Co-authored-by: Taylor Downs <[email protected]>

.gitignore

@@ -53,6 +53,7 @@ benchmarking/*.csv
 monitoring
 
 tmp
+exports
 priv/openfn
 
 .elixir_ls

CHANGELOG.md

@@ -17,6 +17,9 @@ and this project adheres to
 
 ### Added
 
+- Users are now able to export work orders from the History page.
+  [#1698](https://github.com/OpenFn/lightning/issues/1698)
+
 ### Changed
 
 ### Fixed

assets/css/app.css

@@ -115,6 +115,15 @@ div[id^='tippy-'] {
          bg-primary-950;
 }
 
+.icon-button {
+  @apply cursor-pointer
+  hover:text-indigo-500
+  font-bold
+  text-secondary-600
+  transition
+  hover:rotate-[-6deg];
+}
+
 @keyframes fade-in-scale-keys {
   0% {
     scale: 0.95;

lib/lightning/accounts/user_notifier.ex

@@ -12,6 +12,7 @@ defmodule Lightning.Accounts.UserNotifier do
   import Swoosh.Email
 
   alias Lightning.Accounts.User
+  alias Lightning.Helpers
   alias Lightning.Mailer
   alias Lightning.Projects
   alias Lightning.Projects.Project
@@ -272,6 +273,18 @@ defmodule Lightning.Accounts.UserNotifier do
     """)
   end
 
+  def notify_history_export_completion(user, project_file) do
+    deliver(user, "Your OpenFn History Export Is Complete", """
+    Hello #{user.first_name},
+
+    You history export started requested on #{Helpers.format_date(project_file.inserted_at)} is completed. Please visit this URL to download the file:acceptor
+
+    #{url(LightningWeb.Endpoint, ~p"/project_files/#{project_file.id}/download")}
+
+    OpenFn
+    """)
+  end
+
   def build_digest_url(workflow, start_date, end_date) do
     uri_params =
       SearchParams.to_uri_params(%{

lib/lightning/application.ex

@@ -85,32 +85,53 @@ defmodule Lightning.Application do
         Application.get_env(:libcluster, :topologies)
       end
 
-    children = [
-      Lightning.PromEx,
-      {Cluster.Supervisor, [topologies, [name: Lightning.ClusterSupervisor]]},
-      {Lightning.Vault, Application.get_env(:lightning, Lightning.Vault, [])},
-      # Start the Ecto repository
-      Lightning.Repo,
-      # Start Oban,
-      {Oban, oban_opts()},
-      # Start the Telemetry supervisor
-      LightningWeb.Telemetry,
-      # Start the PubSub system
-      {Phoenix.PubSub, name: Lightning.PubSub},
-      {Finch, name: Lightning.Finch},
-      auth_providers_cache_childspec,
-      # Start the Endpoint (http/https)
-      LightningWeb.Endpoint,
-      Lightning.Workflows.Presence,
-      adaptor_registry_childspec,
-      adaptor_service_childspec,
-      {Lightning.TaskWorker, name: :cli_task_worker},
-      {Lightning.Runtime.RuntimeManager,
-       worker_secret: Lightning.Config.worker_secret()},
-      {Lightning.KafkaTriggers.Supervisor, type: :supervisor}
-      # Start a worker by calling: Lightning.Worker.start_link(arg)
-      # {Lightning.Worker, arg}
-    ]
+    goth =
+      Application.get_env(:lightning, Lightning.Google, [])
+      |> then(fn config ->
+        if config[:required] do
+          {Goth,
+           name: Lightning.Google,
+           source:
+             {:service_account, config[:credentials],
+              [
+                scopes: [
+                  "openid",
+                  "https://www.googleapis.com/auth/userinfo.email",
+                  "https://www.googleapis.com/auth/cloud-platform"
+                ]
+              ]}}
+        end
+      end)
+
+    children =
+      [
+        Lightning.PromEx,
+        {Cluster.Supervisor, [topologies, [name: Lightning.ClusterSupervisor]]},
+        {Lightning.Vault, Application.get_env(:lightning, Lightning.Vault, [])},
+        # Start the Ecto repository
+        Lightning.Repo,
+        # Start Oban,
+        {Oban, oban_opts()},
+        goth,
+        # Start the Telemetry supervisor
+        LightningWeb.Telemetry,
+        # Start the PubSub system
+        {Phoenix.PubSub, name: Lightning.PubSub},
+        {Finch, name: Lightning.Finch},
+        auth_providers_cache_childspec,
+        # Start the Endpoint (http/https)
+        LightningWeb.Endpoint,
+        Lightning.Workflows.Presence,
+        adaptor_registry_childspec,
+        adaptor_service_childspec,
+        {Lightning.TaskWorker, name: :cli_task_worker},
+        {Lightning.Runtime.RuntimeManager,
+         worker_secret: Lightning.Config.worker_secret()},
+        {Lightning.KafkaTriggers.Supervisor, type: :supervisor}
+        # Start a worker by calling: Lightning.Worker.start_link(arg)
+        # {Lightning.Worker, arg}
+      ]
+      |> Enum.reject(&is_nil/1)
 
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options

lib/lightning/config.ex

@@ -72,6 +72,12 @@ defmodule Lightning.Config do
       AdapterHelper.adapter(key)
     end
 
+    @impl true
+    def google(key) do
+      Application.get_env(:lightning, Lightning.Google, [])
+      |> Keyword.get(key)
+    end
+
     @impl true
     def check_flag?(flag) do
       Application.get_env(:lightning, flag)
@@ -104,6 +110,17 @@ defmodule Lightning.Config do
       1
     end
 
+    @impl true
+    def storage do
+      Application.get_env(:lightning, Lightning.Storage, [])
+    end
+
+    @impl true
+    def storage(key) do
+      storage()
+      |> Keyword.get(key)
+    end
+
     @impl true
     def usage_tracking_cron_opts do
       opts = usage_tracking()
@@ -163,6 +180,7 @@ defmodule Lightning.Config do
   @callback default_max_run_duration() :: integer()
   @callback email_sender_name() :: String.t()
   @callback get_extension_mod(key :: atom()) :: any()
+  @callback google(key :: atom()) :: any()
   @callback grace_period() :: integer()
   @callback instance_admin_email() :: String.t()
   @callback kafka_duplicate_tracking_retention_seconds() :: integer()
@@ -175,6 +193,8 @@ defmodule Lightning.Config do
   @callback repo_connection_token_signer() :: Joken.Signer.t()
   @callback reset_password_token_validity_in_days() :: integer()
   @callback run_token_signer() :: Joken.Signer.t()
+  @callback storage() :: term()
+  @callback storage(key :: atom()) :: term()
   @callback usage_tracking() :: Keyword.t()
   @callback usage_tracking_cron_opts() :: [Oban.Plugins.Cron.cron_input()]
   @callback worker_secret() :: binary() | nil
@@ -242,6 +262,10 @@ defmodule Lightning.Config do
     impl().get_extension_mod(key)
   end
 
+  def google(key) do
+    impl().google(key)
+  end
+
   def cors_origin do
     impl().cors_origin()
   end
@@ -262,6 +286,14 @@ defmodule Lightning.Config do
     impl().reset_password_token_validity_in_days()
   end
 
+  def storage do
+    impl().storage()
+  end
+
+  def storage(key) do
+    impl().storage(key)
+  end
+
   def usage_tracking_cron_opts do
     impl().usage_tracking_cron_opts()
   end

lib/lightning/config/bootstrap.ex

@@ -216,7 +216,8 @@ defmodule Lightning.Config.Bootstrap do
       queues: [
         scheduler: 1,
         workflow_failures: 1,
-        background: 1
+        background: 1,
+        history_exports: 1
       ]
 
     # https://plausible.io/ is an open-source, privacy-friendly alternative to
@@ -482,9 +483,72 @@ defmodule Lightning.Config.Bootstrap do
       number_of_processors: env!("KAFKA_NUMBER_OF_PROCESSORS", :integer, 1)
 
     # # ==============================================================================
+
+    setup_storage()
+
     :ok
   end
 
+  defp setup_storage do
+    config :lightning, Lightning.Storage,
+      path: env!("STORAGE_PATH", :string, ".")
+
+    env!("STORAGE_BACKEND", :string, "local")
+    |> case do
+      "gcs" ->
+        config :lightning, Lightning.Storage,
+          backend: Lightning.Storage.GCS,
+          bucket:
+            env!("GCS_BUCKET", :string, nil) ||
+              raise("GCS_BUCKET is not set, but STORAGE_BACKEND is set to gcs")
+
+        google_required()
+
+      "local" ->
+        config :lightning, Lightning.Storage, backend: Lightning.Storage.Local
+
+      unknown ->
+        raise """
+        Unknown storage backend: #{unknown}
+
+        Currently supported backends are:
+
+        - gcs
+        - local (default)
+        """
+    end
+  end
+
+  # Not really happy about having to put this here, but for some reason
+  # dialyzer thinks that when :error can be matched then {:error, _} can't be.
+  @dialyzer {:no_match, google_required: 0}
+  defp google_required do
+    with value when is_binary(value) <-
+           env!(
+             "GOOGLE_APPLICATION_CREDENTIALS_JSON",
+             :string,
+             {:error,
+              "GOOGLE_APPLICATION_CREDENTIALS_JSON is not set, this is required when using Google Cloud services."}
+           ),
+         {:ok, decoded} <- Base.decode64(value),
+         {:ok, credentials} <- Jason.decode(decoded) do
+      config :lightning, Lightning.Google,
+        credentials: credentials,
+        required: true
+    else
+      {:error, %Jason.DecodeError{} = error} ->
+        raise """
+        Could not decode GOOGLE_APPLICATION_CREDENTIALS_JSON: #{Jason.DecodeError.message(error)}
+        """
+
+      {:error, message} ->
+        raise message
+
+      :error ->
+        raise "Could not decode GOOGLE_APPLICATION_CREDENTIALS_JSON"
+    end
+  end
+
   defp github_config do
     decoded_cert =
       env!(

lib/lightning/config/utils.ex

@@ -6,7 +6,7 @@ defmodule Lightning.Config.Utils do
   @doc """
   Retrieve a value nested in the application environment.
   """
-  @spec get_env([atom()], any()) :: any()
+  @spec get_env([atom()] | atom(), any()) :: any()
   def get_env(_keys, default \\ nil)
 
   def get_env([app, key, item], default) do

lib/lightning/invocation.ex

@@ -374,6 +374,12 @@ defmodule Lightning.Invocation do
     |> Repo.all()
   end
 
+  def search_workorders_for_export_query(%Project{id: project_id}, search_params) do
+    project_id
+    |> base_query_without_preload()
+    |> search_workorders_query(search_params)
+  end
+
   def count_workorders(%Project{id: project_id}, search_params) do
     project_id
     |> base_query_without_preload()

lib/lightning/policies/exports.ex

@@ -0,0 +1,19 @@
+defmodule Lightning.Policies.Exports do
+  @moduledoc """
+  The Bodyguard Policy module for Exports
+  """
+  @behaviour Bodyguard.Policy
+
+  alias Lightning.Accounts.User
+  alias Lightning.Projects
+  alias Lightning.Projects.File, as: ProjectFile
+  alias Lightning.Projects.Project
+
+  @type actions :: :download
+
+  @spec authorize(actions(), User.t(), ProjectFile.t()) ::
+          boolean() | {:error, :forbidden}
+  def authorize(:download, %User{} = user, %ProjectFile{project_id: project_id}) do
+    Projects.member_of?(%Project{id: project_id}, user) or {:error, :forbidden}
+  end
+end

lib/lightning/projects.ex

@@ -2,6 +2,7 @@ defmodule Lightning.Projects do
   @moduledoc """
   The Projects context.
   """
+
   use Oban.Worker,
     queue: :background,
     max_attempts: 1
@@ -854,4 +855,12 @@ defmodule Lightning.Projects do
       if collaborator.email == email, do: collaborator.role
     end)
   end
+
+  def list_project_files(%Project{id: project_id}) do
+    from(pf in __MODULE__.File,
+      where: pf.project_id == ^project_id,
+      preload: [:created_by]
+    )
+    |> Repo.all()
+  end
 end