From 9c15cbfc1013b4eab164570bae503574a494b7bc Mon Sep 17 00:00:00 2001 From: deepsidhu85 Date: Fri, 10 Jan 2025 15:14:17 -0600 Subject: [PATCH] Workflow Execution logidze (#890) * Added logidze log_data column for workflow executions * Added tests and updated blacklisting of columns * Updated counts for log size and version that was missed in previous commit which blacklisted attachments_updated_at column * Fixed rubocop warnings * Regenerated new trigger and migration to whitelist the columns required * Updated to include deleted_at in whitelist and updated migration to create logidze snapshots for existing workflow executions * Updated db schema * Fixed rubocop warning * Fixed sql.squish rubocop warning --- ...2252_add_logidze_to_workflow_executions.rb | 24 +++++++++++++++++++ db/schema.rb | 6 ++++- .../logidze_on_workflow_executions_v01.sql | 6 +++++ .../cancel_service_test.rb | 9 +++++++ .../completion_service_test.rb | 10 ++++++++ .../create_service_test.rb | 6 +++++ .../preparation_service_test.rb | 8 +++++++ 7 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20250109162252_add_logidze_to_workflow_executions.rb create mode 100644 db/triggers/logidze_on_workflow_executions_v01.sql diff --git a/db/migrate/20250109162252_add_logidze_to_workflow_executions.rb b/db/migrate/20250109162252_add_logidze_to_workflow_executions.rb new file mode 100644 index 0000000000..550f6d66d7 --- /dev/null +++ b/db/migrate/20250109162252_add_logidze_to_workflow_executions.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Migration to add Logidze to WorkflowExecutions table +class AddLogidzeToWorkflowExecutions < ActiveRecord::Migration[7.2] + def change + add_column :workflow_executions, :log_data, :jsonb + + reversible do |dir| + dir.up do + create_trigger :logidze_on_workflow_executions, on: :workflow_executions + + execute <<-SQL.squish + UPDATE "workflow_executions" as t SET log_data = logidze_snapshot(to_jsonb(t), 'created_at', '{"run_id","name","state","deleted_at"}', true); + SQL + end + + dir.down do + execute <<~SQL.squish + DROP TRIGGER IF EXISTS "logidze_on_workflow_executions" on "workflow_executions"; + SQL + end + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 348653eed3..ca8c7f3b1e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2025_01_07_194839) do +ActiveRecord::Schema[7.2].define(version: 2025_01_09_162252) do # These are extensions that must be enabled in order to support this database enable_extension "hstore" enable_extension "plpgsql" @@ -334,6 +334,7 @@ t.string "name" t.uuid "namespace_id" t.boolean "cleaned", default: false, null: false + t.jsonb "log_data" t.index ["created_at"], name: "index_workflow_executions_on_created_at" t.index ["namespace_id"], name: "index_workflow_executions_on_namespace_id" t.index ["state"], name: "index_workflow_executions_on_state" @@ -752,6 +753,9 @@ create_trigger :logidze_on_attachments, sql_definition: <<-SQL CREATE TRIGGER logidze_on_attachments BEFORE INSERT OR UPDATE ON public.attachments FOR EACH ROW WHEN ((COALESCE(current_setting('logidze.disabled'::text, true), ''::text) <> 'on'::text)) EXECUTE FUNCTION logidze_logger('null', 'updated_at') SQL + create_trigger :logidze_on_workflow_executions, sql_definition: <<-SQL + CREATE TRIGGER logidze_on_workflow_executions BEFORE INSERT OR UPDATE ON public.workflow_executions FOR EACH ROW WHEN ((COALESCE(current_setting('logidze.disabled'::text, true), ''::text) <> 'on'::text)) EXECUTE FUNCTION logidze_logger('null', 'updated_at', '{run_id,name,state,deleted_at}', 'true') + SQL create_trigger :logidze_on_data_exports, sql_definition: <<-SQL CREATE TRIGGER logidze_on_data_exports BEFORE INSERT OR UPDATE ON public.data_exports FOR EACH ROW WHEN ((COALESCE(current_setting('logidze.disabled'::text, true), ''::text) <> 'on'::text)) EXECUTE FUNCTION logidze_logger('null', 'updated_at') SQL diff --git a/db/triggers/logidze_on_workflow_executions_v01.sql b/db/triggers/logidze_on_workflow_executions_v01.sql new file mode 100644 index 0000000000..5a5b427b1b --- /dev/null +++ b/db/triggers/logidze_on_workflow_executions_v01.sql @@ -0,0 +1,6 @@ +CREATE TRIGGER "logidze_on_workflow_executions" +BEFORE UPDATE OR INSERT ON "workflow_executions" FOR EACH ROW +WHEN (coalesce(current_setting('logidze.disabled', true), '') <> 'on') +-- Parameters: history_size_limit (integer), timestamp_column (text), filtered_columns (text[]), +-- include_columns (boolean), debounce_time_ms (integer) +EXECUTE PROCEDURE logidze_logger(null, 'updated_at', '{run_id,name,state,deleted_at}', true); diff --git a/test/services/workflow_executions/cancel_service_test.rb b/test/services/workflow_executions/cancel_service_test.rb index bd60f2197e..d49acac6df 100644 --- a/test/services/workflow_executions/cancel_service_test.rb +++ b/test/services/workflow_executions/cancel_service_test.rb @@ -50,7 +50,11 @@ def setup test 'cancel initial workflow_execution' do @workflow_execution = workflow_executions(:irida_next_example_new) + @workflow_execution.create_logidze_snapshot! + assert 'initial', @workflow_execution.state + assert_equal 1, @workflow_execution.log_data.version + assert_equal 1, @workflow_execution.log_data.size assert WorkflowExecutions::CancelService.new(@workflow_execution, @user).execute @@ -58,6 +62,11 @@ def setup assert_equal 'canceled', @workflow_execution.reload.state assert @workflow_execution.cleaned? + + @workflow_execution.create_logidze_snapshot! + assert 'canceled', @workflow_execution.state + assert_equal 2, @workflow_execution.log_data.version + assert_equal 2, @workflow_execution.log_data.size end end end diff --git a/test/services/workflow_executions/completion_service_test.rb b/test/services/workflow_executions/completion_service_test.rb index e7b9d55d2f..d3f0ed4f4e 100644 --- a/test/services/workflow_executions/completion_service_test.rb +++ b/test/services/workflow_executions/completion_service_test.rb @@ -252,8 +252,13 @@ def setup # rubocop:disable Metrics/AbcSize,Metrics/MethodLength test 'sample outputs on samples_workflow_executions' do workflow_execution = @workflow_execution_with_samples + workflow_execution.create_logidze_snapshot! + assert 'completing', workflow_execution.state + assert_equal 1, workflow_execution.log_data.version + assert_equal 1, workflow_execution.log_data.size + assert WorkflowExecutions::CompletionService.new(workflow_execution, {}).execute assert_equal 'my_run_id_c', workflow_execution.run_id @@ -298,8 +303,13 @@ def setup # rubocop:disable Metrics/AbcSize,Metrics/MethodLength assert_equal @normal2_output_analysis3_file_blob.filename, output3.filename assert_equal @normal2_output_analysis3_file_blob.checksum, output3.file.checksum + workflow_execution.create_logidze_snapshot! + assert_equal 'completed', workflow_execution.state + assert_equal 2, workflow_execution.log_data.version + assert_equal 2, workflow_execution.log_data.size + assert_no_enqueued_emails assert_nil PublicActivity::Activity.find_by( diff --git a/test/services/workflow_executions/create_service_test.rb b/test/services/workflow_executions/create_service_test.rb index d403715d84..a6c9eefa2e 100644 --- a/test/services/workflow_executions/create_service_test.rb +++ b/test/services/workflow_executions/create_service_test.rb @@ -310,6 +310,12 @@ def setup @workflow_execution = WorkflowExecutions::CreateService.new(@user, workflow_params).execute + @workflow_execution.create_logidze_snapshot! + + assert 'initial', @workflow_execution.state + assert_equal 1, @workflow_execution.log_data.version + assert_equal 1, @workflow_execution.log_data.size + assert_equal test_name, @workflow_execution.name expected_tags = { 'createdBy' => @user.email } assert_equal expected_tags, @workflow_execution.tags diff --git a/test/services/workflow_executions/preparation_service_test.rb b/test/services/workflow_executions/preparation_service_test.rb index 17a38629ec..abeda408b7 100644 --- a/test/services/workflow_executions/preparation_service_test.rb +++ b/test/services/workflow_executions/preparation_service_test.rb @@ -14,6 +14,10 @@ def setup test 'prepare workflow_execution with valid params' do assert @workflow_execution.initial? + @workflow_execution.create_logidze_snapshot! + assert_equal 1, @workflow_execution.log_data.version + assert_equal 1, @workflow_execution.log_data.size + assert_difference -> { ActiveStorage::Attachment.count } => 2 do WorkflowExecutions::PreparationService.new(@workflow_execution, @user, {}).execute end @@ -42,6 +46,10 @@ def setup assert_equal sample1_row, samplesheet_csv[1] assert_equal 'prepared', @workflow_execution.state + + @workflow_execution.create_logidze_snapshot! + assert_equal 2, @workflow_execution.log_data.version + assert_equal 2, @workflow_execution.log_data.size end test 'should return false since chosen pipeline is not executable' do