Merge pull request #861 from joyent/ether/v3-catch-up-with-v2.32,33-c…

…hanges v3: catch up with v2.32,33 changes
joyent · Aug 9, 2019 · 6e26b93 · 6e26b93
2 parents 4e71bec + da0893f
commit 6e26b93
Show file tree

Hide file tree

Showing 16 changed files with 326 additions and 16 deletions.
diff --git a/docs/modules/Conch::Command::merge_validation_results.md b/docs/modules/Conch::Command::merge_validation_results.md
@@ -0,0 +1,20 @@
+# NAME
+
+merge\_validation\_results - collapse duplicate validation\_result rows together
+
+# SYNOPSIS
+
+```
+bin/conch merge_validation_results [long options...]
+
+    -n --dry-run  dry-run (no changes are made)
+    --help        print usage message and exit
+```
+
+# LICENSING
+
+Copyright Joyent, Inc.
+
+This Source Code Form is subject to the terms of the Mozilla Public License,
+v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
+one at [http://mozilla.org/MPL/2.0/](http://mozilla.org/MPL/2.0/).
diff --git a/docs/modules/Conch::DB::Result::Device.md b/docs/modules/Conch::DB::Result::Device.md
@@ -189,6 +189,12 @@ Type: has\_many
 
 Related object: [Conch::DB::Result::ValidationState](../modules/Conch::DB::Result::ValidationState)
 
+## relays
+
+Type: many\_to\_many
+
+Composing rels: ["device\_relay\_connections"](#device_relay_connections) -> relay
+
 ## latest\_report\_data
 
 Returns the JSON-decoded content from the most recent device report.

diff --git a/docs/modules/Conch::DB::Result::Relay.md b/docs/modules/Conch::DB::Result::Relay.md
@@ -101,6 +101,18 @@ Type: has\_many
 
 Related object: [Conch::DB::Result::UserRelayConnection](../modules/Conch::DB::Result::UserRelayConnection)
 
+## devices
+
+Type: many\_to\_many
+
+Composing rels: ["device\_relay\_connections"](#device_relay_connections) -> device
+
+## user\_accounts
+
+Type: many\_to\_many
+
+Composing rels: ["user\_relay\_connections"](#user_relay_connections) -> user\_account
+
 # LICENSING
 
 Copyright Joyent, Inc.

diff --git a/docs/modules/Conch::DB::Result::UserAccount.md b/docs/modules/Conch::DB::Result::UserAccount.md
@@ -122,6 +122,18 @@ Type: has\_many
 
 Related object: [Conch::DB::Result::UserWorkspaceRole](../modules/Conch::DB::Result::UserWorkspaceRole)
 
+## relays
+
+Type: many\_to\_many
+
+Composing rels: ["user\_relay\_connections"](#user_relay_connections) -> relay
+
+## workspaces
+
+Type: many\_to\_many
+
+Composing rels: ["user\_workspace\_roles"](#user_workspace_roles) -> workspace
+
 # METHODS
 
 ## check\_password

diff --git a/docs/modules/Conch::DB::Result::Workspace.md b/docs/modules/Conch::DB::Result::Workspace.md
@@ -82,6 +82,12 @@ Type: many\_to\_many
 
 Composing rels: ["workspace\_racks"](#workspace_racks) -> rack
 
+## user\_accounts
+
+Type: many\_to\_many
+
+Composing rels: ["user\_workspace\_roles"](#user_workspace_roles) -> user\_account
+
 ## TO\_JSON
 
 Include information about the user's role, if available.

diff --git a/docs/modules/index.md b/docs/modules/index.md
@@ -6,6 +6,7 @@
 * [Conch::Command::clean_roles](../modules/Conch::Command::clean_roles)
 * [Conch::Command::create_token](../modules/Conch::Command::create_token)
 * [Conch::Command::create_user](../modules/Conch::Command::create_user)
+* [Conch::Command::merge_validation_results](../modules/Conch::Command::merge_validation_results)
 * [Conch::Command::thin_device_reports](../modules/Conch::Command::thin_device_reports)
 * [Conch::Command::update_validation_plans](../modules/Conch::Command::update_validation_plans)
 * [Conch::Command::workspaces](../modules/Conch::Command::workspaces)

diff --git a/lib/Conch/Command/merge_validation_results.pm b/lib/Conch/Command/merge_validation_results.pm
@@ -0,0 +1,189 @@
+package Conch::Command::merge_validation_results;
+
+=pod
+
+=head1 NAME
+
+merge_validation_results - collapse duplicate validation_result rows together
+
+=head1 SYNOPSIS
+
+    bin/conch merge_validation_results [long options...]
+
+        -n --dry-run  dry-run (no changes are made)
+        --help        print usage message and exit
+
+=cut
+
+use Mojo::Base 'Mojolicious::Command', -signatures;
+use Getopt::Long::Descriptive;
+use Try::Tiny;
+use Data::Page;
+
+has description => 'Collapse duplicate validation_result rows together';
+
+has usage => sub { shift->extract_usage };  # extracts from SYNOPSIS
+
+has 'dry_run';
+
+sub run ($self, @opts) {
+    local @ARGV = @opts;
+    my ($opt, $usage) = describe_options(
+        # the descriptions aren't actually used anymore (mojo uses the synopsis instead)... but
+        # the 'usage' text block can be accessed with $usage->text
+        'merge_validation_results %o',
+        [ 'dry-run|n',      'dry-run (no changes are made)' ],
+        [],
+        [ 'help',           'print usage message and exit', { shortcircuit => 1 } ],
+    );
+
+    # ACHTUNG! only run this after migration 92 is done,
+    # because otherwise these queries will take even longer.
+
+    $self->dry_run($opt->dry_run);
+
+    # enable autoflush
+    my $prev = select(STDOUT); $|++; select($prev);
+
+    say Conch::Time->now, '  working'.($self->dry_run ? ' (in dry-run mode)' : '').'...';
+    my $schema = ($self->dry_run ? $self->app->ro_schema : $self->app->schema);
+
+    say 'At start, there are '
+        .$schema->resultset('validation_result')->count
+        .' validation_result rows.';
+    say '';
+
+    my ($validation_results_deleted, $device_count) = (0)x2;
+
+    # consider each device, oldest devices first, in pages of 100 rows each
+    my $device_rs = $schema->resultset('device')
+        ->active
+        ->rows(100)
+        ->page(1)
+        ->order_by('created');
+
+    foreach my $page (1 .. $device_rs->pager->last_page) {
+        $device_rs = $device_rs->page($page);
+        while (my $device = $device_rs->next) {
+            # we process each device's reports in a separate transaction,
+            # so we can abort and resume without redoing everything all over again
+            try {
+                $validation_results_deleted += $schema->txn_do(sub {
+                    $self->_process_device($device);
+                });
+                ++$device_count;
+            }
+            catch {
+                if (/Rollback failed/) {
+                    local $@ = $_;
+                    die;    # propagate the error
+                }
+                print STDERR "\n", 'aborted processing of device ', $device->id, ': ', $_, "\n";
+            };
+        }
+    }
+
+    say '';
+    say Conch::Time->now, '  done.';
+    say '';
+    say $device_count.' devices processed.';
+    say $validation_results_deleted.' validation_result rows '
+        .($self->dry_run ? 'would be ' : '') .'deleted.';
+    say 'there are now '.$schema->resultset('validation_result')->count.' validation_result rows.';
+}
+
+
+sub _process_device ($self, $device) {
+    print 'device id ', $device->id, ': ';
+
+    my @grouping_cols = qw(device_id hardware_product_id validation_id message hint status category component result_order);
+
+    my $schema = $self->app->schema;
+    my ($group_count, $results_deleted) = (0)x2;
+
+    # find groups of validation_result rows that share identical column
+    # values that we have an index on (where new validation_states point
+    # to existing rows: see the end of Conch::ValidationSystem::run_validation_plan.
+    # Consider these groups in pages of 100 rows each.
+    my $groups_to_merge_rs = $schema->resultset('validation_result')
+        ->columns(\@grouping_cols)
+        ->search(
+            { device_id => $device->id },
+            { '+select' => [{ count => '*', -as => 'count' }] })
+        ->group_by(\@grouping_cols)
+        ->as_subselect_rs
+        ->search({ count => { '>' => 1 } })
+        ->columns(\@grouping_cols)
+        ->order_by(\@grouping_cols)
+        ->rows(100)
+        ->page(1)
+        ->hri;
+
+    # we go through the pages backward so we can delete rows as we go and not
+    # break queries for the other pages.
+    foreach my $page (reverse(1 .. $groups_to_merge_rs->pager->last_page)) {
+        $groups_to_merge_rs = $groups_to_merge_rs->page($page);
+
+        # foreach matching set,
+            # iterate through all matching rows oldest-first
+            # save the oldest one,
+            # delete the rest, updating validation_state_member to point to the oldest.
+        while (my $sample_result = $groups_to_merge_rs->next) {
+            ++$group_count;
+            print '.' if $group_count % 100 == 0;
+
+            # all results in this resultset share the same values and can be collapsed down to
+            # a single result row
+            my $member_rs = $schema->resultset('validation_result')
+                ->search({ $sample_result->%{@grouping_cols} });
+
+            if ($self->dry_run) {
+                $results_deleted += $member_rs->count - 1;
+                next;
+            }
+
+            # this is the validation_result row we keep - the oldest in this group
+            my $oldest_member_id = $member_rs
+                ->order_by('created')
+                ->rows(1)
+                ->hri
+                ->get_column('id')
+                ->single;
+
+            # all validation_state_members pointing to results in this group should instead
+            # reference the oldest result in the group
+            $schema->resultset('validation_state_member')
+                ->search(
+                    {
+                        validation_result_id => { '!=' => $oldest_member_id },
+                        (map +('validation_result.'.$_ => $sample_result->{$_}), @grouping_cols),
+                    },
+                    { join => 'validation_result' },
+                )
+                ->update({ validation_result_id => $oldest_member_id });
+
+            # delete all the newly-orphaned validation_result rows
+            # (this is safe because we don't have a cascade delete on result -> member yet)
+            $results_deleted += $member_rs->search({ id => { '!=' => $oldest_member_id } })->delete;
+        }
+    }
+
+    say ' '.$results_deleted.' validation_results deleted';
+    return $results_deleted;
+}
+
+1;
+__END__
+
+=pod
+
+=head1 LICENSING
+
+Copyright Joyent, Inc.
+
+This Source Code Form is subject to the terms of the Mozilla Public License,
+v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
+one at L<http://mozilla.org/MPL/2.0/>.
+
+=cut
+# vim: set ts=4 sts=4 sw=4 et :
diff --git a/lib/Conch/DB/Result/Device.pm b/lib/Conch/DB/Result/Device.pm
@@ -359,9 +359,19 @@ __PACKAGE__->has_many(
   { cascade_copy => 0, cascade_delete => 0 },
 );
 
+=head2 relays
+
+Type: many_to_many
+
+Composing rels: L</device_relay_connections> -> relay
+
+=cut
+
+__PACKAGE__->many_to_many("relays", "device_relay_connections", "relay");
+
 
 # Created by DBIx::Class::Schema::Loader v0.07049
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iOf4s64d7wK8hyLDdMkJTw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:t9HvAuvB75DLfvSRojlSSw
 
 __PACKAGE__->has_many(
   "active_device_disks",

diff --git a/lib/Conch/DB/Result/Relay.pm b/lib/Conch/DB/Result/Relay.pm
@@ -175,9 +175,29 @@ __PACKAGE__->has_many(
   { cascade_copy => 0, cascade_delete => 0 },
 );
 
+=head2 devices
+
+Type: many_to_many
+
+Composing rels: L</device_relay_connections> -> device
+
+=cut
+
+__PACKAGE__->many_to_many("devices", "device_relay_connections", "device");
+
+=head2 user_accounts
+
+Type: many_to_many
+
+Composing rels: L</user_relay_connections> -> user_account
+
+=cut
+
+__PACKAGE__->many_to_many("user_accounts", "user_relay_connections", "user_account");
+
 
 # Created by DBIx::Class::Schema::Loader v0.07049
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H0EoEEnlWK/jcq27Jrz+bw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xrcJDGh7U5PUiE3GqsWqkQ
 
 __PACKAGE__->add_columns(
     '+deactivated' => { is_serializable => 0 },

diff --git a/lib/Conch/DB/Result/UserAccount.pm b/lib/Conch/DB/Result/UserAccount.pm
@@ -201,9 +201,29 @@ __PACKAGE__->has_many(
   { cascade_copy => 0, cascade_delete => 0 },
 );
 
+=head2 relays
+
+Type: many_to_many
+
+Composing rels: L</user_relay_connections> -> relay
+
+=cut
+
+__PACKAGE__->many_to_many("relays", "user_relay_connections", "relay");
+
+=head2 workspaces
+
+Type: many_to_many
+
+Composing rels: L</user_workspace_roles> -> workspace
+
+=cut
+
+__PACKAGE__->many_to_many("workspaces", "user_workspace_roles", "workspace");
+
 
 # Created by DBIx::Class::Schema::Loader v0.07049
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:V1hIBHgsrrsR+UYtGDMI9g
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vpTbeHmoxnxcjZ7EUUg9yA
 
 use DBIx::Class::PassphraseColumn 0.04 ();
 __PACKAGE__->load_components('PassphraseColumn');

diff --git a/lib/Conch/DB/Result/Workspace.pm b/lib/Conch/DB/Result/Workspace.pm
@@ -173,9 +173,19 @@ Composing rels: L</workspace_racks> -> rack
 
 __PACKAGE__->many_to_many("racks", "workspace_racks", "rack");
 
+=head2 user_accounts
+
+Type: many_to_many
+
+Composing rels: L</user_workspace_roles> -> user_account
+
+=cut
+
+__PACKAGE__->many_to_many("user_accounts", "user_workspace_roles", "user_account");
+
 
 # Created by DBIx::Class::Schema::Loader v0.07049
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:jAQTz8eBEJas87XgWV7xbw
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:UTkb6H9/XmVkYnMTHm2uQw
 
 use experimental 'signatures';
 use Sub::Install;