From 8028905499ebc695521ce5f8458813190d7c5521 Mon Sep 17 00:00:00 2001 From: Sebastian Schoenherr Date: Wed, 13 Mar 2024 15:20:18 +0100 Subject: [PATCH 1/6] Remove parameters when job is deleted or retired (#142) Co-authored-by: Lukas Forer --- .../mapred/api/v2/admin/ArchiveJob.java | 5 ++++ .../mapred/api/v2/jobs/GetJobDetails.java | 4 +++ .../cloudgene/mapred/cron/CleanUpTasks.java | 6 +++++ .../mapred/database/ParameterDao.java | 25 ++++++++++++++++++- 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java b/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java index b77aa627..02f76320 100644 --- a/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java +++ b/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java @@ -1,7 +1,9 @@ package cloudgene.mapred.api.v2.admin; +import cloudgene.mapred.database.ParameterDao; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.restlet.data.Parameter; import org.restlet.data.Status; import org.restlet.representation.Representation; import org.restlet.representation.StringRepresentation; @@ -81,6 +83,9 @@ public Representation get() { job.setState(AbstractJob.STATE_RETIRED); dao.update(job); + ParameterDao parameterDao = new ParameterDao(getDatabase()); + parameterDao.deleteAllByJob(job); + if (externalWorkspace != null) { try { externalWorkspace.delete(job.getId()); diff --git a/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java b/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java index 4b816b2e..520ec3bb 100644 --- a/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java +++ b/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java @@ -3,6 +3,7 @@ import java.util.List; import java.util.Vector; +import cloudgene.mapred.database.ParameterDao; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.restlet.data.MediaType; @@ -148,6 +149,9 @@ public Representation deleteJob(Representation entity) { job.setState(AbstractJob.STATE_DELETED); dao.update(job); + ParameterDao parameterDao = new ParameterDao(getDatabase()); + parameterDao.deleteAllByJob(job); + Settings settings = getSettings(); IExternalWorkspace externalWorkspace = null; diff --git a/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java b/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java index c2e56589..f3cb97e3 100644 --- a/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java +++ b/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java @@ -3,6 +3,7 @@ import java.io.File; import java.util.List; +import cloudgene.mapred.database.ParameterDao; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -25,6 +26,8 @@ public class CleanUpTasks { public static int executeRetire(Database database, Settings settings) { JobDao dao = new JobDao(database); + ParameterDao parameterDao = new ParameterDao(database); + List oldJobs = dao.findAllNotifiedJobs(); int deleted = 0; @@ -53,6 +56,9 @@ public static int executeRetire(Database database, Settings settings) { job.setState(AbstractJob.STATE_RETIRED); dao.update(job); + + parameterDao.deleteAllByJob(job); + log.info("Job " + job.getId() + " retired."); deleted++; diff --git a/src/main/java/cloudgene/mapred/database/ParameterDao.java b/src/main/java/cloudgene/mapred/database/ParameterDao.java index eb2decc2..6028b8b8 100644 --- a/src/main/java/cloudgene/mapred/database/ParameterDao.java +++ b/src/main/java/cloudgene/mapred/database/ParameterDao.java @@ -183,7 +183,30 @@ public CloudgeneParameterOutput findById(int id) { } } - class ParameterInputMapper implements IRowMapper { + public boolean deleteAllByJob(AbstractJob job) { + try { + + StringBuilder sql = new StringBuilder(); + sql.append("delete "); + sql.append("from parameter "); + sql.append("where job_id = ?"); + + Object[] params = new Object[1]; + params[0] = job.getId(); + + update(sql.toString(), params); + + log.debug("delete all parameters by job_id '" + job.getId() + "' successful."); + + return true; + + } catch (SQLException e) { + log.error("delete all parameters by job_id '" + job.getId() + "' failed.", e); + return false; + } + } + + class ParameterInputMapper implements IRowMapper { @Override public Object mapRow(ResultSet rs, int row) throws SQLException { From 28723741a785bcbdd68741cdc9c062a36416cb23 Mon Sep 17 00:00:00 2001 From: Sebastian Schoenherr Date: Wed, 13 Mar 2024 15:26:22 +0100 Subject: [PATCH 2/6] Prepare release v2.8.3 --- pom.xml | 2 +- src/main/html/webapp/package-lock.json | 4 ++-- src/main/html/webapp/package.json | 2 +- src/main/java/cloudgene/mapred/Main.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index a9b3ec47..bb453a52 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 cloudgene cloudgene - 2.8.2 + 2.8.3 Cloudgene http://www.cloudgene.io diff --git a/src/main/html/webapp/package-lock.json b/src/main/html/webapp/package-lock.json index 20c5e85a..83f2d96d 100644 --- a/src/main/html/webapp/package-lock.json +++ b/src/main/html/webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "cloudgene", - "version": "2.8.2", + "version": "2.8.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cloudgene", - "version": "2.8.2", + "version": "2.8.3", "license": "AGPL-3.0", "dependencies": { "@fortawesome/fontawesome-free": "5.3.1", diff --git a/src/main/html/webapp/package.json b/src/main/html/webapp/package.json index 235fa51c..989041f9 100644 --- a/src/main/html/webapp/package.json +++ b/src/main/html/webapp/package.json @@ -1,6 +1,6 @@ { "name": "cloudgene", - "version": "2.8.2", + "version": "2.8.3", "description": "Cloudgene web client", "homepage": "http://www.cloudgene.io", "author": "Lukas Forer Date: Thu, 21 Mar 2024 16:19:32 +0100 Subject: [PATCH 3/6] Fixe job names (#145) * Show job name in dialogs instead of id * Add waiting jobs to counters * Revert deletion of parameters --- .../components/core/job/detail/detail.js | 6 ++--- .../webapp/components/core/job/list/list.js | 4 +-- .../mapred/api/v2/admin/ArchiveJob.java | 3 --- .../mapred/api/v2/jobs/GetJobDetails.java | 3 --- .../mapred/api/v2/server/GetCounter.java | 8 ++++-- .../cloudgene/mapred/cron/CleanUpTasks.java | 4 --- .../cloudgene/mapred/database/JobDao.java | 25 +++++++++++++++++++ .../mapred/database/ParameterDao.java | 23 ----------------- .../cloudgene/mapred/database/UserDao.java | 23 ++++++++++++++++- 9 files changed, 58 insertions(+), 41 deletions(-) diff --git a/src/main/html/webapp/components/core/job/detail/detail.js b/src/main/html/webapp/components/core/job/detail/detail.js index 5ef45138..147973c7 100644 --- a/src/main/html/webapp/components/core/job/detail/detail.js +++ b/src/main/html/webapp/components/core/job/detail/detail.js @@ -76,7 +76,7 @@ export default Control.extend({ '#delete-btn click': function(el, ev) { var that = this; - bootbox.confirm("Are you sure you want to delete " + that.job.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to delete " + that.job.attr('name') + "?", function(result) { if (result) { var okButton = $("button[data-bb-handler='confirm']"); @@ -106,7 +106,7 @@ export default Control.extend({ '#cancel-btn click': function(el, ev) { var that = this; - bootbox.confirm("Are you sure you want to cancel " + that.job.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to cancel " + that.job.attr('name') + "?", function(result) { if (result) { var okButton = $("button[data-bb-handler='confirm']"); @@ -136,7 +136,7 @@ export default Control.extend({ '#restart-btn click': function(el, ev) { var that = this; - bootbox.confirm("Are you sure you want to restart " + that.job.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to restart " + that.job.attr('name') + "?", function(result) { if (result) { var okButton = $("button[data-bb-handler='confirm']"); diff --git a/src/main/html/webapp/components/core/job/list/list.js b/src/main/html/webapp/components/core/job/list/list.js index d78b4f0d..63f886bf 100644 --- a/src/main/html/webapp/components/core/job/list/list.js +++ b/src/main/html/webapp/components/core/job/list/list.js @@ -48,7 +48,7 @@ export default Control.extend({ var card = $(el).closest('.card'); var job = domData.get.call(card[0], 'job'); - bootbox.confirm("Are you sure you want to delete " + job.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to delete " + job.attr('name') + "?", function(result) { if (result) { var okButton = $("button[data-bb-handler='confirm']"); @@ -78,7 +78,7 @@ export default Control.extend({ var card = $(el).closest('.card'); var job = domData.get.call(card[0], 'job'); - bootbox.confirm("Are you sure you want to cancel " + job.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to cancel " + job.attr('name') + "?", function(result) { if (result) { var okButton = $("button[data-bb-handler='confirm']"); diff --git a/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java b/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java index 02f76320..e97c6f30 100644 --- a/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java +++ b/src/main/java/cloudgene/mapred/api/v2/admin/ArchiveJob.java @@ -83,9 +83,6 @@ public Representation get() { job.setState(AbstractJob.STATE_RETIRED); dao.update(job); - ParameterDao parameterDao = new ParameterDao(getDatabase()); - parameterDao.deleteAllByJob(job); - if (externalWorkspace != null) { try { externalWorkspace.delete(job.getId()); diff --git a/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java b/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java index 520ec3bb..da456e1b 100644 --- a/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java +++ b/src/main/java/cloudgene/mapred/api/v2/jobs/GetJobDetails.java @@ -149,9 +149,6 @@ public Representation deleteJob(Representation entity) { job.setState(AbstractJob.STATE_DELETED); dao.update(job); - ParameterDao parameterDao = new ParameterDao(getDatabase()); - parameterDao.deleteAllByJob(job); - Settings settings = getSettings(); IExternalWorkspace externalWorkspace = null; diff --git a/src/main/java/cloudgene/mapred/api/v2/server/GetCounter.java b/src/main/java/cloudgene/mapred/api/v2/server/GetCounter.java index 7656aaaa..bccf597b 100644 --- a/src/main/java/cloudgene/mapred/api/v2/server/GetCounter.java +++ b/src/main/java/cloudgene/mapred/api/v2/server/GetCounter.java @@ -2,6 +2,7 @@ import java.util.Map; +import cloudgene.mapred.database.JobDao; import org.restlet.representation.Representation; import org.restlet.representation.StringRepresentation; import org.restlet.resource.Get; @@ -41,10 +42,13 @@ public Representation get() { for (String key : counters.keySet()) { jsonWaiting.put(key, counters.get(key)); } + JobDao jobDao = new JobDao(getDatabase()); + int waitingJobs = jobDao.findAllByState(AbstractJob.STATE_WAITING).size(); + jsonWaiting.put("runs", waitingJobs); jsonCounters.put("waiting", jsonWaiting); - UserDao dao = new UserDao(getDatabase()); - jsonCounters.put("users", dao.findAll().size()); + UserDao dao = new UserDao(getDatabase()); + jsonCounters.put("users", dao.countAll()); return new StringRepresentation(jsonCounters.toString()); diff --git a/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java b/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java index f3cb97e3..cdc10cf2 100644 --- a/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java +++ b/src/main/java/cloudgene/mapred/cron/CleanUpTasks.java @@ -26,8 +26,6 @@ public class CleanUpTasks { public static int executeRetire(Database database, Settings settings) { JobDao dao = new JobDao(database); - ParameterDao parameterDao = new ParameterDao(database); - List oldJobs = dao.findAllNotifiedJobs(); int deleted = 0; @@ -57,8 +55,6 @@ public static int executeRetire(Database database, Settings settings) { dao.update(job); - parameterDao.deleteAllByJob(job); - log.info("Job " + job.getId() + " retired."); deleted++; diff --git a/src/main/java/cloudgene/mapred/database/JobDao.java b/src/main/java/cloudgene/mapred/database/JobDao.java index ac65f39f..78723a05 100644 --- a/src/main/java/cloudgene/mapred/database/JobDao.java +++ b/src/main/java/cloudgene/mapred/database/JobDao.java @@ -412,6 +412,31 @@ public List findAllByState(int state) { } } + @SuppressWarnings("unchecked") + public int countAllByState(int state) { + + StringBuilder sql = new StringBuilder(); + sql.append("select count(*) "); + sql.append("from job "); + sql.append("where state = ? "); + + Object[] params = new Object[1]; + params[0] = state; + + int result = 0; + + try { + result = (Integer) queryForObject(sql.toString(), params, new IntegerMapper()); + log.debug("count all old jobs successful. results: " + result); + + return result; + } catch (SQLException e) { + log.error("count all old jobs failed", e); + return 0; + } + } + + public AbstractJob findById(String id) { return findById(id, true); diff --git a/src/main/java/cloudgene/mapred/database/ParameterDao.java b/src/main/java/cloudgene/mapred/database/ParameterDao.java index 6028b8b8..e6251463 100644 --- a/src/main/java/cloudgene/mapred/database/ParameterDao.java +++ b/src/main/java/cloudgene/mapred/database/ParameterDao.java @@ -183,29 +183,6 @@ public CloudgeneParameterOutput findById(int id) { } } - public boolean deleteAllByJob(AbstractJob job) { - try { - - StringBuilder sql = new StringBuilder(); - sql.append("delete "); - sql.append("from parameter "); - sql.append("where job_id = ?"); - - Object[] params = new Object[1]; - params[0] = job.getId(); - - update(sql.toString(), params); - - log.debug("delete all parameters by job_id '" + job.getId() + "' successful."); - - return true; - - } catch (SQLException e) { - log.error("delete all parameters by job_id '" + job.getId() + "' failed.", e); - return false; - } - } - class ParameterInputMapper implements IRowMapper { @Override diff --git a/src/main/java/cloudgene/mapred/database/UserDao.java b/src/main/java/cloudgene/mapred/database/UserDao.java index 37af8957..fd831317 100644 --- a/src/main/java/cloudgene/mapred/database/UserDao.java +++ b/src/main/java/cloudgene/mapred/database/UserDao.java @@ -208,7 +208,28 @@ public List findAll() { } return result; } - + + @SuppressWarnings("unchecked") + public int countAll() { + + StringBuilder sql = new StringBuilder(); + sql.append("select count(*) "); + sql.append("from `user` "); + + int result = 0; + + try { + result = (Integer) queryForObject(sql.toString(), new IntegerMapper()); + log.debug("count all users successful. results: " + result); + + return result; + } catch (SQLException e) { + log.error("count all users failed", e); + return 0; + } + } + + @SuppressWarnings("unchecked") public List findByQuery(String query) { From fd247f771d0037714ca836fbaf9ec3791b586508 Mon Sep 17 00:00:00 2001 From: Sebastian Schoenherr Date: Fri, 22 Mar 2024 11:48:55 +0100 Subject: [PATCH 4/6] Prepare release v2.8.4 --- pom.xml | 2 +- src/main/html/webapp/package-lock.json | 4 ++-- src/main/html/webapp/package.json | 2 +- src/main/java/cloudgene/mapred/Main.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index bb453a52..30158533 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 cloudgene cloudgene - 2.8.3 + 2.8.4 Cloudgene http://www.cloudgene.io diff --git a/src/main/html/webapp/package-lock.json b/src/main/html/webapp/package-lock.json index 83f2d96d..2fec2908 100644 --- a/src/main/html/webapp/package-lock.json +++ b/src/main/html/webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "cloudgene", - "version": "2.8.3", + "version": "2.8.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cloudgene", - "version": "2.8.3", + "version": "2.8.4", "license": "AGPL-3.0", "dependencies": { "@fortawesome/fontawesome-free": "5.3.1", diff --git a/src/main/html/webapp/package.json b/src/main/html/webapp/package.json index 989041f9..74e33de5 100644 --- a/src/main/html/webapp/package.json +++ b/src/main/html/webapp/package.json @@ -1,6 +1,6 @@ { "name": "cloudgene", - "version": "2.8.3", + "version": "2.8.4", "description": "Cloudgene web client", "homepage": "http://www.cloudgene.io", "author": "Lukas Forer Date: Sat, 23 Mar 2024 11:01:10 +0100 Subject: [PATCH 5/6] Add queue size to counters --- .../mapred/api/v2/server/GetCounter.java | 20 +++++++++---------- .../cloudgene/mapred/jobs/WorkflowEngine.java | 4 ++++ .../cloudgene/mapred/jobs/queue/Queue.java | 4 ++++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/main/java/cloudgene/mapred/api/v2/server/GetCounter.java b/src/main/java/cloudgene/mapred/api/v2/server/GetCounter.java index bccf597b..5c7fee97 100644 --- a/src/main/java/cloudgene/mapred/api/v2/server/GetCounter.java +++ b/src/main/java/cloudgene/mapred/api/v2/server/GetCounter.java @@ -18,11 +18,11 @@ public class GetCounter extends BaseResource { public Representation get() { JSONObject jsonCounters = new JSONObject(); - + // complete Map counters = getWorkflowEngine().getCounters( - AbstractJob.STATE_SUCCESS); - JSONObject jsonComplete = new JSONObject(); + AbstractJob.STATE_SUCCESS); + JSONObject jsonComplete = new JSONObject(); for (String key : counters.keySet()) { jsonComplete.put(key, counters.get(key)); } @@ -30,7 +30,7 @@ public Representation get() { // running counters = getWorkflowEngine().getCounters(AbstractJob.STATE_RUNNING); - JSONObject jsonRunning = new JSONObject(); + JSONObject jsonRunning = new JSONObject(); for (String key : counters.keySet()) { jsonRunning.put(key, counters.get(key)); } @@ -38,19 +38,19 @@ public Representation get() { // waiting counters = getWorkflowEngine().getCounters(AbstractJob.STATE_WAITING); - JSONObject jsonWaiting= new JSONObject(); + JSONObject jsonWaiting= new JSONObject(); for (String key : counters.keySet()) { jsonWaiting.put(key, counters.get(key)); } - JobDao jobDao = new JobDao(getDatabase()); - int waitingJobs = jobDao.findAllByState(AbstractJob.STATE_WAITING).size(); - jsonWaiting.put("runs", waitingJobs); jsonCounters.put("waiting", jsonWaiting); UserDao dao = new UserDao(getDatabase()); jsonCounters.put("users", dao.countAll()); - - + + JSONObject queue = new JSONObject(); + queue.put("size", getWorkflowEngine().getSize()); + jsonCounters.put("queue", queue); + return new StringRepresentation(jsonCounters.toString()); } diff --git a/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java b/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java index 8e8664c4..a21ac21a 100644 --- a/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java +++ b/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java @@ -241,6 +241,10 @@ public List getAllJobsInLongTimeQueue() { return jobs; } + public int getSize() { + return longTimeQueue.getSize() + shortTimeQueue.getSize(); + } + class SetupThread extends PriorityRunnable { private AbstractJob job; diff --git a/src/main/java/cloudgene/mapred/jobs/queue/Queue.java b/src/main/java/cloudgene/mapred/jobs/queue/Queue.java index 3a5386bf..20c7df49 100644 --- a/src/main/java/cloudgene/mapred/jobs/queue/Queue.java +++ b/src/main/java/cloudgene/mapred/jobs/queue/Queue.java @@ -305,6 +305,10 @@ public boolean isInQueue(AbstractJob job) { } } + public int getSize() { + return queue.size(); + } + protected class PriorityComparator implements Comparator { @Override From a8ded7a7fd8d5ff2ed9794d9b5e3050645697594 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 23 Mar 2024 17:47:49 +0100 Subject: [PATCH 6/6] Prepare release 2.8.5 --- pom.xml | 2 +- src/main/html/webapp/package-lock.json | 4 ++-- src/main/html/webapp/package.json | 2 +- src/main/java/cloudgene/mapred/Main.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 30158533..c09e5114 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 cloudgene cloudgene - 2.8.4 + 2.8.5 Cloudgene http://www.cloudgene.io diff --git a/src/main/html/webapp/package-lock.json b/src/main/html/webapp/package-lock.json index 2fec2908..00d7445a 100644 --- a/src/main/html/webapp/package-lock.json +++ b/src/main/html/webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "cloudgene", - "version": "2.8.4", + "version": "2.8.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cloudgene", - "version": "2.8.4", + "version": "2.8.5", "license": "AGPL-3.0", "dependencies": { "@fortawesome/fontawesome-free": "5.3.1", diff --git a/src/main/html/webapp/package.json b/src/main/html/webapp/package.json index 74e33de5..122698cd 100644 --- a/src/main/html/webapp/package.json +++ b/src/main/html/webapp/package.json @@ -1,6 +1,6 @@ { "name": "cloudgene", - "version": "2.8.4", + "version": "2.8.5", "description": "Cloudgene web client", "homepage": "http://www.cloudgene.io", "author": "Lukas Forer