From 5be67da664889446bf6376ed12feafcdd82f7d7b Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 30 Nov 2020 09:17:37 +0000 Subject: [PATCH 1/4] Pending workers don't have a meadow_host yet --- modules/Bio/EnsEMBL/Hive/Worker.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Bio/EnsEMBL/Hive/Worker.pm b/modules/Bio/EnsEMBL/Hive/Worker.pm index e42414f45..bef67f7db 100644 --- a/modules/Bio/EnsEMBL/Hive/Worker.pm +++ b/modules/Bio/EnsEMBL/Hive/Worker.pm @@ -471,7 +471,7 @@ sub toString { $include_analysis ? ( 'analysis='.($current_role ? $current_role->analysis->logic_name.'('.$current_role->analysis_id.')' : 'UNSPECIALIZED') ) : (), 'resource_class_id='.($self->resource_class_id // 'NULL'), 'meadow='.$self->meadow_type.'/'.$self->meadow_name, - 'process='.$self->meadow_user.'@'.$self->meadow_host.'#'.$self->process_id, + 'process='.$self->meadow_user.($self->meadow_host ? '@'.$self->meadow_host : 'UNALLOCATED').'#'.$self->process_id, 'when_checked_in='.($self->when_checked_in // 'NEVER'), 'batch_size='.($current_role ? $current_role->analysis->stats->get_or_estimate_batch_size() : 'UNSPECIALIZED'), 'job_limit='.($self->job_limiter->available_capacity() // 'NONE'), From c28c7efea9b4cd858e1f647ec96a213f61600d58 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Tue, 1 Dec 2020 11:39:56 +0000 Subject: [PATCH 2/4] Added the example of a new view to show how jobs (analyses) are related The view is not yet enabled and will require matching schema patches. --- sql/procedures.mysql | 33 ++++++++++++++++++++++++++++++++- sql/procedures.pgsql | 33 ++++++++++++++++++++++++++++++++- sql/procedures.sqlite | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 97 insertions(+), 3 deletions(-) diff --git a/sql/procedures.mysql b/sql/procedures.mysql index a7c6d6ac7..1a875e9b7 100644 --- a/sql/procedures.mysql +++ b/sql/procedures.mysql @@ -32,7 +32,7 @@ CONTACT -- -- Usage: -- select * from progress; # the whole table (may take ages to generate, depending on the size of your pipeline) --- select * from progress where logic_name like 'family_blast%'; # only show family_blast-related analyses +-- select * from progress where analysis_name_and_id like 'family_blast%'; # only show family_blast-related analyses -- select * from progress where retry_count>1; # only show jobs that have been tried more than once CREATE OR REPLACE VIEW progress AS @@ -49,6 +49,37 @@ CREATE OR REPLACE VIEW progress AS ORDER BY a.analysis_id, j.status; +-- show the hierarchy of hive jobs (which analysis seeded jobs where) ---------------------------- +-- +-- Usage: +-- select * from job_hierarchy; # the whole table (may take ages to generate, depending on the size of your pipeline) +-- select * from job_hierarchy where retry_count>1; # only show jobs that have been tried more than once + +CREATE OR REPLACE VIEW job_hierarchy AS + SELECT + CONCAT(a1.logic_name, '(', a1.analysis_id, ')') AS analysis_name_and_id, + CONCAT(a2.logic_name, '(', a2.analysis_id, ')') AS parent_analysis_name_and_id, + j1.status AS status, + j1.retry_count AS retry_count, + COUNT(0) AS cnt, + MIN(j1.job_id) AS example_job_id + FROM + job j1 + LEFT JOIN job j2 ON (j1.prev_job_id = j2.job_id) + LEFT JOIN analysis_base a1 ON (j1.analysis_id = a1.analysis_id) + LEFT JOIN analysis_base a2 ON (j2.analysis_id = a2.analysis_id) + GROUP BY + a1.logic_name, + a2.logic_name, + j1.status, + j1.retry_count + ORDER BY + a1.analysis_id, + a2.analysis_id, + j1.retry_count; +DROP VIEW job_hierarchy; + + -- a convenient view that also incorporates (otherwise redundant) analysis_id and logic_name ------------ -- -- Usage: diff --git a/sql/procedures.pgsql b/sql/procedures.pgsql index 656e03166..54b8dc058 100644 --- a/sql/procedures.pgsql +++ b/sql/procedures.pgsql @@ -32,7 +32,7 @@ CONTACT -- -- Usage: -- select * from progress; # the whole table (may take ages to generate, depending on the size of your pipeline) --- select * from progress where logic_name like 'family_blast%'; # only show family_blast-related analyses +-- select * from progress where analysis_name_and_id like 'family_blast%'; # only show family_blast-related analyses -- select * from progress where retry_count>1; # only show jobs that have been tried more than once CREATE OR REPLACE VIEW progress AS @@ -49,6 +49,37 @@ CREATE OR REPLACE VIEW progress AS ORDER BY a.analysis_id, j.status; +-- show the hierarchy of hive jobs (which analysis seeded jobs where) ---------------------------- +-- +-- Usage: +-- select * from job_hierarchy; # the whole table (may take ages to generate, depending on the size of your pipeline) +-- select * from job_hierarchy where retry_count>1; # only show jobs that have been tried more than once + +CREATE OR REPLACE VIEW job_hierarchy AS + SELECT + a1.logic_name || '(' || a1.analysis_id || ')' AS analysis_name_and_id, + a2.logic_name || '(' || a2.analysis_id || ')' AS parent_analysis_name_and_id, + j1.status AS status, + j1.retry_count AS retry_count, + COUNT(0) AS cnt, + MIN(j1.job_id) AS example_job_id + FROM + job j1 + LEFT JOIN job j2 ON (j1.prev_job_id = j2.job_id) + LEFT JOIN analysis_base a1 ON (j1.analysis_id = a1.analysis_id) + LEFT JOIN analysis_base a2 ON (j2.analysis_id = a2.analysis_id) + GROUP BY + a1.logic_name, + a2.logic_name, + j1.status, + j1.retry_count + ORDER BY + a1.analysis_id, + a2.analysis_id, + j1.retry_count; +DROP VIEW job_hierarchy; + + -- a convenient view that also incorporates (otherwise redundant) analysis_id and logic_name ------------ -- -- Usage: diff --git a/sql/procedures.sqlite b/sql/procedures.sqlite index 0e163401a..230350893 100644 --- a/sql/procedures.sqlite +++ b/sql/procedures.sqlite @@ -32,7 +32,7 @@ CONTACT -- -- Usage: -- select * from progress; # the whole table (may take ages to generate, depending on the size of your pipeline) --- select * from progress where logic_name like 'family_blast%'; # only show family_blast-related analyses +-- select * from progress where analysis_name_and_id like 'family_blast%'; # only show family_blast-related analyses -- select * from progress where retry_count>1; # only show jobs that have been tried more than once DROP VIEW IF EXISTS progress; @@ -50,6 +50,38 @@ CREATE VIEW IF NOT EXISTS progress AS ORDER BY a.analysis_id, j.status; +-- show the hierarchy of hive jobs (which analysis seeded jobs where) ---------------------------- +-- +-- Usage: +-- select * from job_hierarchy; # the whole table (may take ages to generate, depending on the size of your pipeline) +-- select * from job_hierarchy where retry_count>1; # only show jobs that have been tried more than once + +DROP VIEW IF EXISTS job_hierarchy; +CREATE VIEW IF NOT EXISTS job_hierarchy AS + SELECT + a1.logic_name || '(' || a1.analysis_id || ')' AS analysis_name_and_id, + a2.logic_name || '(' || a2.analysis_id || ')' AS parent_analysis_name_and_id, + j1.status AS status, + j1.retry_count AS retry_count, + COUNT(0) AS cnt, + MIN(j1.job_id) AS example_job_id + FROM + job j1 + LEFT JOIN job j2 ON (j1.prev_job_id = j2.job_id) + LEFT JOIN analysis_base a1 ON (j1.analysis_id = a1.analysis_id) + LEFT JOIN analysis_base a2 ON (j2.analysis_id = a2.analysis_id) + GROUP BY + a1.logic_name, + a2.logic_name, + j1.status, + j1.retry_count + ORDER BY + a1.analysis_id, + a2.analysis_id, + j1.retry_count; +DROP VIEW job_hierarchy; + + -- a convenient view that also incorporates (otherwise redundant) analysis_id and logic_name ------------ -- -- Usage: From c09a06ed41f9deb3e21b364898d0fba6d3b00074 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 14 Dec 2020 13:22:15 +0000 Subject: [PATCH 3/4] Moved the housekeeping .t tests into their own directory --- t/{ => 00.housekeeping}/apache2.t | 2 +- t/{ => 00.housekeeping}/ascii.t | 2 +- t/{ => 00.housekeeping}/perlcritic.t | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename t/{ => 00.housekeeping}/apache2.t (98%) rename t/{ => 00.housekeeping}/ascii.t (96%) rename t/{ => 00.housekeeping}/perlcritic.t (95%) diff --git a/t/apache2.t b/t/00.housekeeping/apache2.t similarity index 98% rename from t/apache2.t rename to t/00.housekeeping/apache2.t index 1f517fefa..4af2a92f5 100755 --- a/t/apache2.t +++ b/t/00.housekeeping/apache2.t @@ -38,7 +38,7 @@ my $original_dir = cwd(); chdir($file_dir); my $cur_dir = cwd(); chdir($original_dir); -my $root = File::Spec->catdir($cur_dir, File::Spec->updir()); +my $root = File::Spec->catdir($cur_dir, File::Spec->updir(), File::Spec->updir()); =head2 has_apache2_licence diff --git a/t/ascii.t b/t/00.housekeeping/ascii.t similarity index 96% rename from t/ascii.t rename to t/00.housekeeping/ascii.t index 9374d5ce8..52d7c9ca4 100755 --- a/t/ascii.t +++ b/t/00.housekeeping/ascii.t @@ -40,7 +40,7 @@ my $original_dir = cwd(); chdir($file_dir); my $cur_dir = cwd(); chdir($original_dir); -my $root = File::Spec->catdir($cur_dir, File::Spec->updir()); +my $root = File::Spec->catdir($cur_dir, File::Spec->updir(), File::Spec->updir()); sub is_ascii { diff --git a/t/perlcritic.t b/t/00.housekeeping/perlcritic.t similarity index 95% rename from t/perlcritic.t rename to t/00.housekeeping/perlcritic.t index 1bbfc6613..bd196cd26 100755 --- a/t/perlcritic.t +++ b/t/00.housekeeping/perlcritic.t @@ -46,7 +46,7 @@ my $original_dir = cwd(); chdir($file_dir); my $cur_dir = cwd(); chdir($original_dir); -my $root = File::Spec->catdir($cur_dir, File::Spec->updir()); +my $root = File::Spec->catdir($cur_dir, File::Spec->updir(), File::Spec->updir()); # Configure critic Test::Perl::Critic->import(-profile => File::Spec->catfile($root, 'perlcriticrc'), -severity => 5, -verbose => 8); From 5bca20e72cdff37f5dbb653117db3b0640de42bb Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Tue, 15 Dec 2020 00:07:47 +0000 Subject: [PATCH 4/4] These two scripts should be executable --- t/03.scripts/peekJob_output.t | 0 t/03.scripts/tweak_pipeline_output.t | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 t/03.scripts/peekJob_output.t mode change 100644 => 100755 t/03.scripts/tweak_pipeline_output.t diff --git a/t/03.scripts/peekJob_output.t b/t/03.scripts/peekJob_output.t old mode 100644 new mode 100755 diff --git a/t/03.scripts/tweak_pipeline_output.t b/t/03.scripts/tweak_pipeline_output.t old mode 100644 new mode 100755