diff --git a/docs/modules/Conch::Command::merge_validation_results.md b/docs/modules/Conch::Command::merge_validation_results.md new file mode 100644 index 000000000..996d013cd --- /dev/null +++ b/docs/modules/Conch::Command::merge_validation_results.md @@ -0,0 +1,20 @@ +# NAME + +merge\_validation\_results - collapse duplicate validation\_result rows together + +# SYNOPSIS + +``` +bin/conch merge_validation_results [long options...] + + -n --dry-run dry-run (no changes are made) + --help print usage message and exit +``` + +# LICENSING + +Copyright Joyent, Inc. + +This Source Code Form is subject to the terms of the Mozilla Public License, +v.2.0. If a copy of the MPL was not distributed with this file, You can obtain +one at [http://mozilla.org/MPL/2.0/](http://mozilla.org/MPL/2.0/). diff --git a/docs/modules/Conch::DB::Result::Device.md b/docs/modules/Conch::DB::Result::Device.md index f4f4736b2..7d603b2f5 100644 --- a/docs/modules/Conch::DB::Result::Device.md +++ b/docs/modules/Conch::DB::Result::Device.md @@ -189,6 +189,12 @@ Type: has\_many Related object: [Conch::DB::Result::ValidationState](../modules/Conch::DB::Result::ValidationState) +## relays + +Type: many\_to\_many + +Composing rels: ["device\_relay\_connections"](#device_relay_connections) -> relay + ## latest\_report\_data Returns the JSON-decoded content from the most recent device report. diff --git a/docs/modules/Conch::DB::Result::Relay.md b/docs/modules/Conch::DB::Result::Relay.md index 76659840c..cc844f0af 100644 --- a/docs/modules/Conch::DB::Result::Relay.md +++ b/docs/modules/Conch::DB::Result::Relay.md @@ -101,6 +101,18 @@ Type: has\_many Related object: [Conch::DB::Result::UserRelayConnection](../modules/Conch::DB::Result::UserRelayConnection) +## devices + +Type: many\_to\_many + +Composing rels: ["device\_relay\_connections"](#device_relay_connections) -> device + +## user\_accounts + +Type: many\_to\_many + +Composing rels: ["user\_relay\_connections"](#user_relay_connections) -> user\_account + # LICENSING Copyright Joyent, Inc. diff --git a/docs/modules/Conch::DB::Result::UserAccount.md b/docs/modules/Conch::DB::Result::UserAccount.md index b1fb12255..723ebbaab 100644 --- a/docs/modules/Conch::DB::Result::UserAccount.md +++ b/docs/modules/Conch::DB::Result::UserAccount.md @@ -122,6 +122,18 @@ Type: has\_many Related object: [Conch::DB::Result::UserWorkspaceRole](../modules/Conch::DB::Result::UserWorkspaceRole) +## relays + +Type: many\_to\_many + +Composing rels: ["user\_relay\_connections"](#user_relay_connections) -> relay + +## workspaces + +Type: many\_to\_many + +Composing rels: ["user\_workspace\_roles"](#user_workspace_roles) -> workspace + # METHODS ## check\_password diff --git a/docs/modules/Conch::DB::Result::Workspace.md b/docs/modules/Conch::DB::Result::Workspace.md index 797f21315..675edaf49 100644 --- a/docs/modules/Conch::DB::Result::Workspace.md +++ b/docs/modules/Conch::DB::Result::Workspace.md @@ -82,6 +82,12 @@ Type: many\_to\_many Composing rels: ["workspace\_racks"](#workspace_racks) -> rack +## user\_accounts + +Type: many\_to\_many + +Composing rels: ["user\_workspace\_roles"](#user_workspace_roles) -> user\_account + ## TO\_JSON Include information about the user's role, if available. diff --git a/docs/modules/index.md b/docs/modules/index.md index c662002fc..8fdc9a70d 100644 --- a/docs/modules/index.md +++ b/docs/modules/index.md @@ -6,6 +6,7 @@ * [Conch::Command::clean_roles](../modules/Conch::Command::clean_roles) * [Conch::Command::create_token](../modules/Conch::Command::create_token) * [Conch::Command::create_user](../modules/Conch::Command::create_user) +* [Conch::Command::merge_validation_results](../modules/Conch::Command::merge_validation_results) * [Conch::Command::thin_device_reports](../modules/Conch::Command::thin_device_reports) * [Conch::Command::update_validation_plans](../modules/Conch::Command::update_validation_plans) * [Conch::Command::workspaces](../modules/Conch::Command::workspaces) diff --git a/lib/Conch/Command/merge_validation_results.pm b/lib/Conch/Command/merge_validation_results.pm new file mode 100644 index 000000000..60e6e1aca --- /dev/null +++ b/lib/Conch/Command/merge_validation_results.pm @@ -0,0 +1,189 @@ +package Conch::Command::merge_validation_results; + +=pod + +=head1 NAME + +merge_validation_results - collapse duplicate validation_result rows together + +=head1 SYNOPSIS + + bin/conch merge_validation_results [long options...] + + -n --dry-run dry-run (no changes are made) + --help print usage message and exit + +=cut + +use Mojo::Base 'Mojolicious::Command', -signatures; +use Getopt::Long::Descriptive; +use Try::Tiny; +use Data::Page; + +has description => 'Collapse duplicate validation_result rows together'; + +has usage => sub { shift->extract_usage }; # extracts from SYNOPSIS + +has 'dry_run'; + +sub run ($self, @opts) { + local @ARGV = @opts; + my ($opt, $usage) = describe_options( + # the descriptions aren't actually used anymore (mojo uses the synopsis instead)... but + # the 'usage' text block can be accessed with $usage->text + 'merge_validation_results %o', + [ 'dry-run|n', 'dry-run (no changes are made)' ], + [], + [ 'help', 'print usage message and exit', { shortcircuit => 1 } ], + ); + + # ACHTUNG! only run this after migration 92 is done, + # because otherwise these queries will take even longer. + + $self->dry_run($opt->dry_run); + + # enable autoflush + my $prev = select(STDOUT); $|++; select($prev); + + say Conch::Time->now, ' working'.($self->dry_run ? ' (in dry-run mode)' : '').'...'; + my $schema = ($self->dry_run ? $self->app->ro_schema : $self->app->schema); + + say 'At start, there are ' + .$schema->resultset('validation_result')->count + .' validation_result rows.'; + say ''; + + my ($validation_results_deleted, $device_count) = (0)x2; + + # consider each device, oldest devices first, in pages of 100 rows each + my $device_rs = $schema->resultset('device') + ->active + ->rows(100) + ->page(1) + ->order_by('created'); + + foreach my $page (1 .. $device_rs->pager->last_page) { + $device_rs = $device_rs->page($page); + while (my $device = $device_rs->next) { + # we process each device's reports in a separate transaction, + # so we can abort and resume without redoing everything all over again + try { + $validation_results_deleted += $schema->txn_do(sub { + $self->_process_device($device); + }); + ++$device_count; + } + catch { + if (/Rollback failed/) { + local $@ = $_; + die; # propagate the error + } + print STDERR "\n", 'aborted processing of device ', $device->id, ': ', $_, "\n"; + }; + } + } + + say ''; + say Conch::Time->now, ' done.'; + say ''; + say $device_count.' devices processed.'; + say $validation_results_deleted.' validation_result rows ' + .($self->dry_run ? 'would be ' : '') .'deleted.'; + say 'there are now '.$schema->resultset('validation_result')->count.' validation_result rows.'; +} + + +sub _process_device ($self, $device) { + print 'device id ', $device->id, ': '; + + my @grouping_cols = qw(device_id hardware_product_id validation_id message hint status category component result_order); + + my $schema = $self->app->schema; + my ($group_count, $results_deleted) = (0)x2; + + # find groups of validation_result rows that share identical column + # values that we have an index on (where new validation_states point + # to existing rows: see the end of Conch::ValidationSystem::run_validation_plan. + # Consider these groups in pages of 100 rows each. + my $groups_to_merge_rs = $schema->resultset('validation_result') + ->columns(\@grouping_cols) + ->search( + { device_id => $device->id }, + { '+select' => [{ count => '*', -as => 'count' }] }) + ->group_by(\@grouping_cols) + ->as_subselect_rs + ->search({ count => { '>' => 1 } }) + ->columns(\@grouping_cols) + ->order_by(\@grouping_cols) + ->rows(100) + ->page(1) + ->hri; + + # we go through the pages backward so we can delete rows as we go and not + # break queries for the other pages. + foreach my $page (reverse(1 .. $groups_to_merge_rs->pager->last_page)) { + $groups_to_merge_rs = $groups_to_merge_rs->page($page); + + # foreach matching set, + # iterate through all matching rows oldest-first + # save the oldest one, + # delete the rest, updating validation_state_member to point to the oldest. + while (my $sample_result = $groups_to_merge_rs->next) { + ++$group_count; + print '.' if $group_count % 100 == 0; + + # all results in this resultset share the same values and can be collapsed down to + # a single result row + my $member_rs = $schema->resultset('validation_result') + ->search({ $sample_result->%{@grouping_cols} }); + + if ($self->dry_run) { + $results_deleted += $member_rs->count - 1; + next; + } + + # this is the validation_result row we keep - the oldest in this group + my $oldest_member_id = $member_rs + ->order_by('created') + ->rows(1) + ->hri + ->get_column('id') + ->single; + + # all validation_state_members pointing to results in this group should instead + # reference the oldest result in the group + $schema->resultset('validation_state_member') + ->search( + { + validation_result_id => { '!=' => $oldest_member_id }, + (map +('validation_result.'.$_ => $sample_result->{$_}), @grouping_cols), + }, + { join => 'validation_result' }, + ) + ->update({ validation_result_id => $oldest_member_id }); + + # delete all the newly-orphaned validation_result rows + # (this is safe because we don't have a cascade delete on result -> member yet) + $results_deleted += $member_rs->search({ id => { '!=' => $oldest_member_id } })->delete; + } + } + + say ' '.$results_deleted.' validation_results deleted'; + return $results_deleted; +} + +1; +__END__ + +=pod + +=head1 LICENSING + +Copyright Joyent, Inc. + +This Source Code Form is subject to the terms of the Mozilla Public License, +v.2.0. If a copy of the MPL was not distributed with this file, You can obtain +one at L. + +=cut +# vim: set ts=4 sts=4 sw=4 et : diff --git a/lib/Conch/DB/Result/Device.pm b/lib/Conch/DB/Result/Device.pm index db45dba6f..9436c36e9 100644 --- a/lib/Conch/DB/Result/Device.pm +++ b/lib/Conch/DB/Result/Device.pm @@ -359,9 +359,19 @@ __PACKAGE__->has_many( { cascade_copy => 0, cascade_delete => 0 }, ); +=head2 relays + +Type: many_to_many + +Composing rels: L -> relay + +=cut + +__PACKAGE__->many_to_many("relays", "device_relay_connections", "relay"); + # Created by DBIx::Class::Schema::Loader v0.07049 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iOf4s64d7wK8hyLDdMkJTw +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:t9HvAuvB75DLfvSRojlSSw __PACKAGE__->has_many( "active_device_disks", diff --git a/lib/Conch/DB/Result/Relay.pm b/lib/Conch/DB/Result/Relay.pm index 981f57fcc..b14c9a8f9 100644 --- a/lib/Conch/DB/Result/Relay.pm +++ b/lib/Conch/DB/Result/Relay.pm @@ -175,9 +175,29 @@ __PACKAGE__->has_many( { cascade_copy => 0, cascade_delete => 0 }, ); +=head2 devices + +Type: many_to_many + +Composing rels: L -> device + +=cut + +__PACKAGE__->many_to_many("devices", "device_relay_connections", "device"); + +=head2 user_accounts + +Type: many_to_many + +Composing rels: L -> user_account + +=cut + +__PACKAGE__->many_to_many("user_accounts", "user_relay_connections", "user_account"); + # Created by DBIx::Class::Schema::Loader v0.07049 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H0EoEEnlWK/jcq27Jrz+bw +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xrcJDGh7U5PUiE3GqsWqkQ __PACKAGE__->add_columns( '+deactivated' => { is_serializable => 0 }, diff --git a/lib/Conch/DB/Result/UserAccount.pm b/lib/Conch/DB/Result/UserAccount.pm index 894d7ca9c..6e95bc9ad 100644 --- a/lib/Conch/DB/Result/UserAccount.pm +++ b/lib/Conch/DB/Result/UserAccount.pm @@ -201,9 +201,29 @@ __PACKAGE__->has_many( { cascade_copy => 0, cascade_delete => 0 }, ); +=head2 relays + +Type: many_to_many + +Composing rels: L -> relay + +=cut + +__PACKAGE__->many_to_many("relays", "user_relay_connections", "relay"); + +=head2 workspaces + +Type: many_to_many + +Composing rels: L -> workspace + +=cut + +__PACKAGE__->many_to_many("workspaces", "user_workspace_roles", "workspace"); + # Created by DBIx::Class::Schema::Loader v0.07049 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:V1hIBHgsrrsR+UYtGDMI9g +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vpTbeHmoxnxcjZ7EUUg9yA use DBIx::Class::PassphraseColumn 0.04 (); __PACKAGE__->load_components('PassphraseColumn'); diff --git a/lib/Conch/DB/Result/Workspace.pm b/lib/Conch/DB/Result/Workspace.pm index c1af4dd06..20d2bacff 100644 --- a/lib/Conch/DB/Result/Workspace.pm +++ b/lib/Conch/DB/Result/Workspace.pm @@ -173,9 +173,19 @@ Composing rels: L -> rack __PACKAGE__->many_to_many("racks", "workspace_racks", "rack"); +=head2 user_accounts + +Type: many_to_many + +Composing rels: L -> user_account + +=cut + +__PACKAGE__->many_to_many("user_accounts", "user_workspace_roles", "user_account"); + # Created by DBIx::Class::Schema::Loader v0.07049 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:jAQTz8eBEJas87XgWV7xbw +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:UTkb6H9/XmVkYnMTHm2uQw use experimental 'signatures'; use Sub::Install; diff --git a/lib/Conch/ValidationSystem.pm b/lib/Conch/ValidationSystem.pm index 6ed9257af..c828682c8 100644 --- a/lib/Conch/ValidationSystem.pm +++ b/lib/Conch/ValidationSystem.pm @@ -354,6 +354,9 @@ sub run_validation_plan ($self, %options) { $_->%{qw(message hint status category component)}, }), $validator->validation_results; + + $self->log->debug('validation '.$validation->name.' returned no results for device id '.$device->id) + if not $validator->validation_results; } # maybe no validations ran? this is a problem. @@ -370,6 +373,9 @@ sub run_validation_plan ($self, %options) { return ($status, @validation_results) if $options{no_save_db}; + $self->log->debug('recording validation status '.$status.' with ' + .(scalar @validation_results).' results for device id '.$device->id); + return $self->schema->resultset('validation_state')->create({ device_id => $device->id, device_report_id => $device_report->id, diff --git a/schema-loader.yaml b/schema-loader.yaml index 643db64fd..ce692793b 100644 --- a/schema-loader.yaml +++ b/schema-loader.yaml @@ -15,6 +15,8 @@ loader_options: result_base_class: Conch::DB::Result + allow_extra_m2m_cols: 1 + rel_name_map: user: user_account DeviceNeighbor: diff --git a/sql/migrations/0100-assert-safe-for-v3.sql b/sql/migrations/0100-assert-safe-for-v3.sql index 2bc9250f0..470b13330 100644 --- a/sql/migrations/0100-assert-safe-for-v3.sql +++ b/sql/migrations/0100-assert-safe-for-v3.sql @@ -1,5 +1,5 @@ do $$ begin -assert (select max(id) from migration) = 91, 'not all v2 migrations have been run; cannot proceed with v3 upgrade'; +assert (select max(id) from migration) = 93, 'not all v2 migrations have been run; cannot proceed with v3 upgrade'; end; $$; SELECT run_migration(100, ''); diff --git a/sql/migrations/0102-drop-uuid-ossp.sql b/sql/migrations/0102-drop-uuid-ossp.sql deleted file mode 100644 index 5886c0a21..000000000 --- a/sql/migrations/0102-drop-uuid-ossp.sql +++ /dev/null @@ -1,11 +0,0 @@ -SELECT run_migration(102, $$ - - alter table validation alter column id set default gen_random_uuid(); - alter table validation_plan alter column id set default gen_random_uuid(); - alter table validation_result alter column id set default gen_random_uuid(); - alter table validation_state alter column id set default gen_random_uuid(); - alter table workspace alter column id set default gen_random_uuid(); - - drop extension "uuid-ossp"; - -$$); diff --git a/sql/schema.sql b/sql/schema.sql index 07036c516..15ca782fc 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -1339,6 +1339,13 @@ CREATE INDEX validation_plan_member_validation_plan_id_idx ON public.validation_ CREATE UNIQUE INDEX validation_plan_name_idx ON public.validation_plan USING btree (name) WHERE (deactivated IS NULL); +-- +-- Name: validation_result_all_columns_idx; Type: INDEX; Schema: public; Owner: conch +-- + +CREATE INDEX validation_result_all_columns_idx ON public.validation_result USING btree (device_id, hardware_product_id, validation_id, message, hint, status, category, component, result_order); + + -- -- Name: validation_result_device_id_idx; Type: INDEX; Schema: public; Owner: conch --