From e7d41e41791812653fa13f2354edb685e39b181c Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Thu, 1 Aug 2024 08:15:10 -0400 Subject: [PATCH 01/57] first passthru of tSNE image rendering by display tile --- www/api/resources/tsne_data.py | 40 +++++++++++++++++++++++----------- www/css/expression.css | 2 +- www/css/projection.css | 2 +- www/js/classes/tilegrid.js | 9 ++++++++ 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/www/api/resources/tsne_data.py b/www/api/resources/tsne_data.py index bbd271f7..23444000 100644 --- a/www/api/resources/tsne_data.py +++ b/www/api/resources/tsne_data.py @@ -4,7 +4,7 @@ import os import re import sys -from math import ceil +from math import ceil, log2 from pathlib import Path import geardb @@ -45,13 +45,13 @@ def __init__(self, message="") -> None: super().__init__(self.message) -def calculate_figure_height(num_plots): +def calculate_figure_height(num_plots, span=1): """Determine height of tsne plot based on number of group elements.""" - return (num_plots * 2) + (num_plots -1) + return ((num_plots * 2) + (num_plots - 1)) * span -def calculate_figure_width(num_plots): +def calculate_figure_width(num_plots, span=1): """Determine width of tsne plot based on number of group elements.""" - return (num_plots * 6) + (num_plots -1) + return ((num_plots * 6) + (num_plots - 1)) * span def calculate_num_legend_cols(group_len): """Determine number of columns legend should have in tSNE plot.""" @@ -239,6 +239,7 @@ def post(self, dataset_id): projection_id = req.get('projection_id', None) # projection id of csv output colorblind_mode = req.get('colorblind_mode', False) high_dpi = req.get('high_dpi', False) + grid_spec = req.get('grid_spec', "1/1/1/1") # start_row/start_col/end_row/end_col (end not inclusive) sc.settings.figdir = '/tmp/' if not dataset_id: @@ -479,7 +480,7 @@ def post(self, dataset_id): elif color_idx_name in selected.obs: # Alternative method. Associate with hexcodes already stored in the dataframe # Making the assumption that these values are hexcodes - grouped = selected.obs.groupby([colorize_by, color_idx_name]) + grouped = selected.obs.groupby([colorize_by, color_idx_name], observed=False) # Ensure one-to-one mapping between category and hexcodes if len(selected.obs[colorize_by].unique()) == len(grouped): # Test if names are color hexcodes and use those if applicable (if first is good, assume all are) @@ -544,9 +545,23 @@ def post(self, dataset_id): io_fig = sc.pl.embedding(selected, **kwargs) ax = io_fig.get_axes() + # break grid_spec into spans + grid_spec = grid_spec.split('/') + grid_spec = [int(x) for x in grid_spec] + row_span = grid_spec[2] - grid_spec[0] + col_span = grid_spec[3] - grid_spec[1] + row_span_pixels = row_span * 360 # number of pixels this row span will take up + col_span_pixels = col_span * 90 # number of pixels this col span will take up + # set the figsize based on the number of plots - io_fig.set_figheight(calculate_figure_height(num_plots)) - io_fig.set_figwidth(calculate_figure_width(num_plots)) + #io_fig.set_figheight(calculate_figure_height(num_plots, row_span)) + #io_fig.set_figwidth(calculate_figure_width(num_plots, col_span)) + + # Set the figsize (in inches) + dpi = io_fig.dpi # default dpi is 100, but will be saved as 150 later on + # With 2 plots as a default (gene expression and colorize_by), we want to grow the figure size slowly based on the number of plots + io_fig.set_figheight(row_span_pixels * log2(num_plots) / dpi) + io_fig.set_figwidth(col_span_pixels * log2(num_plots) / dpi) # rename axes labels if type(ax) == list: @@ -583,15 +598,14 @@ def post(self, dataset_id): selected.file.close() with io.BytesIO() as io_pic: - # ? From what I'm reading and seeing, this line does not seem to make a difference if bbox_inches is set to "tight" - io_fig.tight_layout() # This crops out much of the whitespace around the plot. The "savefig" line does this with the legend too - # Set the saved figure dpi based on the number of observations in the dataset after filtering if high_dpi: dpi = max(150, int(df.shape[0] / 100)) sc.settings.set_figure_params(dpi_save=dpi) - # if high_dpi, double the figsize height - io_fig.set_figheight(calculate_figure_height(num_plots) * 2) + # Figure height is calculated based on the number of plots + io_fig.set_figheight(calculate_figure_height(num_plots)) + io_fig.set_figwidth(calculate_figure_width(num_plots)) + io_fig.savefig(io_pic, format='png', bbox_inches="tight") else: # Moved this to the end to prevent any issues with the dpi setting diff --git a/www/css/expression.css b/www/css/expression.css index a2d9be6e..3c60177f 100644 --- a/www/css/expression.css +++ b/www/css/expression.css @@ -75,8 +75,8 @@ ul#go-terms li { width: 1080px; /* grid width + 2*border width */ } +/* tSNE plots */ .js-tile .card-image img { - object-fit: fill; width: fit-content; height: 100%; margin: auto; diff --git a/www/css/projection.css b/www/css/projection.css index fb0a1f3a..b609cbfe 100644 --- a/www/css/projection.css +++ b/www/css/projection.css @@ -64,8 +64,8 @@ ul#go-terms li { width: 1080px; /* grid width + 2*border width */ } +/* tSNE plots */ .js-tile .card-image img { - object-fit: fill; width: fit-content; height: 100%; margin: auto; diff --git a/www/js/classes/tilegrid.js b/www/js/classes/tilegrid.js index 73ef90ad..613a46e9 100644 --- a/www/js/classes/tilegrid.js +++ b/www/js/classes/tilegrid.js @@ -1459,6 +1459,11 @@ class DatasetTile { const plotType = display.plot_type; const plotConfig = display.plotly_config; + const tileElement = document.getElementById(`tile-${this.tile.tileId}`); + if (!this.isZoomed) { + plotConfig.grid_spec = tileElement.style.gridArea // add grid spec to plot config + } + const plotContainer = document.querySelector(`#tile-${this.tile.tileId} .card-image`); if (!plotContainer) return; // tile was removed before data was returned plotContainer.replaceChildren(); // erase plot @@ -1512,6 +1517,10 @@ class DatasetTile { const plotConfig = JSON.parse(JSON.stringify(display.plotly_config)); plotConfig.high_dpi = true; + const tileElement = document.getElementById(`tile-${this.tile.tileId}`); + if (!this.isZoomed) { + plotConfig.grid_spec = tileElement.style.gridArea // add grid spec to plot config + } const data = await apiCallsMixin.fetchTsneImage(datasetId, analysisObj, plotType, plotConfig); if (data?.success < 1) { From bb655f10260f43fdaea43e58be6fe39ac8c66b3d Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Wed, 7 Aug 2024 15:18:42 -0400 Subject: [PATCH 02/57] removing unused function arg --- bin/profile_single_heatmap_run.py | 2 +- bin/profile_single_projectr_tsne_run.py | 2 +- lib/geardb.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/profile_single_heatmap_run.py b/bin/profile_single_heatmap_run.py index 1040b5b6..5a046282 100644 --- a/bin/profile_single_heatmap_run.py +++ b/bin/profile_single_heatmap_run.py @@ -147,7 +147,7 @@ def get_analysis(analysis, dataset_id, session_id): ana.type = analysis['type'] except: user = geardb.get_user_from_session_id(session_id) - ana.discover_type(current_user_id=user.id) + ana.discover_type() else: ds = geardb.Dataset(id=dataset_id, has_h5ad=1) h5_path = ds.get_file_path() diff --git a/bin/profile_single_projectr_tsne_run.py b/bin/profile_single_projectr_tsne_run.py index 42fa1566..cb5552f3 100644 --- a/bin/profile_single_projectr_tsne_run.py +++ b/bin/profile_single_projectr_tsne_run.py @@ -183,7 +183,7 @@ def get_analysis(analysis, dataset_id, session_id): ana.type = analysis['type'] except: user = geardb.get_user_from_session_id(session_id) - ana.discover_type(current_user_id=user.id) + ana.discover_type() else: ds = geardb.Dataset(id=dataset_id, has_h5ad=1) h5_path = ds.get_file_path() diff --git a/lib/geardb.py b/lib/geardb.py index 2866229d..ed4aa855 100644 --- a/lib/geardb.py +++ b/lib/geardb.py @@ -104,7 +104,7 @@ def get_analysis(analysis, dataset_id, session_id): if 'type' in analysis: ana.type = analysis['type'] else: - ana.discover_type(current_user_id=user_id) + ana.discover_type() else: ds = Dataset(id=dataset_id, has_h5ad=1) h5_path = ds.get_file_path() @@ -682,7 +682,7 @@ def discover_vetting(self, current_user_id=None): return 'community' - def discover_type(self, current_user_id=None): + def discover_type(self): """ Given an analysis ID it's technically possible to scan the directory hierarchies and find the type. From 92f47f3e787446713df22426036cb2bba9cce11f Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Wed, 7 Aug 2024 15:43:36 -0400 Subject: [PATCH 03/57] implemented #646 --- www/cgi/download_source_file.cgi | 34 +++++++++++++++++--------------- www/js/classes/analysis-ui.js | 1 + www/js/classes/analysis.js | 34 ++++++++++++++++++++++++++++++-- www/js/classes/tilegrid.js | 19 ++++++++++++++---- www/js/dataset_explorer.js | 25 +++++++++++++---------- www/js/sc_workbench.js | 7 +++++++ www/sc_workbench.html | 16 +++++++++++++++ 7 files changed, 104 insertions(+), 32 deletions(-) diff --git a/www/cgi/download_source_file.cgi b/www/cgi/download_source_file.cgi index 64ad09e2..61d7779c 100755 --- a/www/cgi/download_source_file.cgi +++ b/www/cgi/download_source_file.cgi @@ -17,11 +17,28 @@ import geardb def main(): form = cgi.FieldStorage() dataset_id = html.escape(form.getvalue('dataset_id')) + analysis_id = html.escape(form.getvalue('analysis_id', "")) + session_id = html.escape(form.getvalue('session_id', "")) dtype = html.escape(form.getvalue('type')) dataset = geardb.Dataset(id=dataset_id) tarball_path = dataset.get_tarball_path() h5ad_path = dataset.get_file_path() + # if analysis ID is passed, retrieve the h5ad file for the analysis to download + if analysis_id: + + # Need session id to get "user_unsaved" analyses + if not session_id: + session_id = None + + analysis = geardb.Analysis(id=analysis_id, dataset_id=dataset_id, session_id=session_id) + analysis.discover_type() + try: + h5ad_path = analysis.dataset_path() + except Exception as e: + print(str(e), file=sys.stderr) + h5ad_path = "" + if dtype == 'tarball' and os.path.isfile(tarball_path): print("Content-type: application/octet-stream") print("Content-Disposition: attachment; filename={0}.tar.gz".format(dataset_id)) @@ -41,22 +58,7 @@ def main(): copyfileobj(binfile, sys.stdout.buffer) else: - result_error = "Requested file could not be found. Unable to download data file." - - print("Content-type: text/html") - print() - print(""" - - - - - - -

Error: {0}

-

Redirecting... Click here if you are not redirected - - - """.format(result_error, 'http://gear.igs.umaryland.edu')) + raise FileNotFoundError("File not found") if __name__ == '__main__': main() diff --git a/www/js/classes/analysis-ui.js b/www/js/classes/analysis-ui.js index 488dfc9b..dff5f5d7 100644 --- a/www/js/classes/analysis-ui.js +++ b/www/js/classes/analysis-ui.js @@ -34,6 +34,7 @@ class AnalysisUI { analysisPublicElt = "#analyses-public" analysisRenameElts = ".js-show-rename-input" analysisDeleteElts = ".js-delete-analysis" + analysisDownloadElts = ".js-download-analysis" analysisActionContainer = "#analysis-action-c" btnSaveAnalysisElt = "#btn-save-analysis" btnDeleteUnsavedAnalysisElt = "#btn-delete-unsaved-analysis" diff --git a/www/js/classes/analysis.js b/www/js/classes/analysis.js index afc518ed..390633d3 100644 --- a/www/js/classes/analysis.js +++ b/www/js/classes/analysis.js @@ -177,6 +177,34 @@ class Analysis { } } + async download() { + + // create URL parameters + const params = { + session_id: this.userSessionId, + dataset_id: this.dataset.id, + analysis_id: this.id, + type: "h5ad", + } + + // download the h5ad + const url = `./cgi/download_source_file.cgi?${new URLSearchParams(params).toString()}`; + try { + const {data} = await axios.get(url, {responseType: 'blob'}); + const blob = new Blob([data], {type: 'application/octet-stream'}); + const downloadUrl = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = downloadUrl; + a.download = `${this.dataset.id}.${this.id}.h5ad`; + a.click(); + } catch (error) { + console.error("Error downloading analysis h5ad", this); + logErrorInConsole(error); + createToast(`Error downloading analysis h5ad`); + } + + } + /** * Retrieves the stored analysis data from the server. * @returns {Promise} A promise that resolves when the analysis data is retrieved. @@ -518,9 +546,11 @@ class Analysis { const imgSrc = response.request.responseURL; const html = `${title}`; document.querySelector(target).innerHTML = html; - } else { - console.error(`Error: ${response.status}`); + return; } + + console.error(`Error: ${response.status}`); + createToast(`Error getting analysis image`); } /** diff --git a/www/js/classes/tilegrid.js b/www/js/classes/tilegrid.js index 73ef90ad..81c4aded 100644 --- a/www/js/classes/tilegrid.js +++ b/www/js/classes/tilegrid.js @@ -751,8 +751,14 @@ class DatasetTile { case "download-bundle": // Download dataset bundle if (hasTarball && isDownloadable) { - const url = `./cgi/download_source_file.cgi?type=tarball&dataset_id=${datasetId}`; - item.href = url; + try { + const url = `./cgi/download_source_file.cgi?type=tarball&dataset_id=${datasetId}`; + item.href = url; + } catch (error) { + logErrorInConsole(error); + createToast("An error occurred while trying to download the dataset bundle."); + } + } else { item.classList.add("is-hidden"); } @@ -760,8 +766,13 @@ class DatasetTile { case "download-h5ad": // Download h5ad file if (hasH5ad && isDownloadable) { - const url = `./cgi/download_source_file.cgi?type=h5ad&dataset_id=${datasetId}`; - item.href = url; + try { + const url = `./cgi/download_source_file.cgi?type=h5ad&dataset_id=${datasetId}`; + item.href = url; + } catch (error) { + logErrorInConsole(error); + createToast("An error occurred while trying to download the h5ad file."); + } } else { item.classList.add("is-hidden"); } diff --git a/www/js/dataset_explorer.js b/www/js/dataset_explorer.js index f34a282a..ccc56fa2 100644 --- a/www/js/dataset_explorer.js +++ b/www/js/dataset_explorer.js @@ -291,16 +291,21 @@ const addDatasetListEventListeners = () => { for (const classElt of document.getElementsByClassName("js-download-dataset")) { classElt.addEventListener("click", async (e) => { - // download the h5ad - const datasetId = e.currentTarget.dataset.datasetId; - const url = `./cgi/download_source_file.cgi?type=h5ad&dataset_id=${datasetId}`; - const {data} = await axios.get(url, {responseType: 'blob'}); - const blob = new Blob([data], {type: 'application/octet-stream'}); - const downloadUrl = window.URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = downloadUrl; - a.download = `${datasetId}.h5ad`; - a.click(); + try { + // download the h5ad + const datasetId = e.currentTarget.dataset.datasetId; + const url = `./cgi/download_source_file.cgi?type=h5ad&dataset_id=${datasetId}`; + const {data} = await axios.get(url, {responseType: 'blob'}); + const blob = new Blob([data], {type: 'application/octet-stream'}); + const downloadUrl = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = downloadUrl; + a.download = `${datasetId}.h5ad`; + a.click(); + } catch (error) { + logErrorInConsole(error); + createToast("Failed to download dataset"); + } }); } diff --git a/www/js/sc_workbench.js b/www/js/sc_workbench.js index d66a40ff..e69c9451 100644 --- a/www/js/sc_workbench.js +++ b/www/js/sc_workbench.js @@ -616,6 +616,13 @@ for (const button of document.querySelectorAll(UI.analysisDeleteElts)) { }) } +for (const button of document.querySelectorAll(UI.analysisDownloadElts)) { + button.addEventListener("click", (event) => { + // Download the current analysis + currentAnalysis.download(); + }); +} + // Show the "rename" analysis label input when the button is clicked for (const button of document.querySelectorAll(UI.analysisRenameElts)) { button.addEventListener("click", (event) => { diff --git a/www/sc_workbench.html b/www/sc_workbench.html index 8b434d1b..dd3089c6 100644 --- a/www/sc_workbench.html +++ b/www/sc_workbench.html @@ -212,6 +212,14 @@

Initial composition plots
+

+ +

+

+ +

+

+

+ +

+
+
+
+ `; + + // append element to DOM to get its dimensions + document.body.appendChild(popoverContent); + + const arrowElement = document.getElementById('arrow'); + + // Create popover (help from https://floating-ui.com/docs/tutorial) + computePosition(button, popoverContent, { + placement: 'bottom', + middleware: [ + flip(), // flip to top if there is not enough space on butotm + shift(), // shift the popover to the right if there is not enough space on the left + offset(5), // offset relative to the button + arrow({ element: arrowElement }) // add an arrow pointing to the button + ], + }).then(({ x, y, placement, middlewareData }) => { + console.log('Popover position:', x, y, placement, middlewareData); + // Position the popover + Object.assign(popoverContent.style, { + left: `${x}px`, + top: `${y}px`, + }); + // Accessing the data + const { x: arrowX, y: arrowY } = middlewareData.arrow; + + // Position the arrow relative to the popover + const staticSide = { + top: 'bottom', + right: 'left', + bottom: 'top', + left: 'right', + }[placement.split('-')[0]]; + + // Set the arrow position + Object.assign(arrowElement.style, { + left: arrowX != null ? `${arrowX}px` : '', + top: arrowY != null ? `${arrowY}px` : '', + right: '', + bottom: '', + [staticSide]: '-4px', + }); + }); + + // Add event listener to cancel button + document.getElementById('cancel-analysis-delete').addEventListener('click', () => { + popoverContent.remove(); + }); + + // Add event listener to confirm button + document.getElementById('confirm-analysis-delete').addEventListener('click', async (event) => { + event.target.classList.add("is-loading"); + + try { + // Delete the current analysis + currentAnalysis.delete(); + } catch (error) { + logErrorInConsole(error); + createToast("Failed to delete dataset"); + } finally { + event.target.classList.remove("is-loading"); + popoverContent.remove(); + } + }); + }); +} + + // Handle the analysis selection change document.querySelector(UI.analysisSelect).addEventListener("change", async (event) => { diff --git a/www/js/stepper-fxns.js b/www/js/stepper-fxns.js index a9c81b35..d0b8258b 100644 --- a/www/js/stepper-fxns.js +++ b/www/js/stepper-fxns.js @@ -158,9 +158,9 @@ const openNextSteps = (selectors, activeSelector=null) => { const resetStepperWithHrefs = (activeSelectorHref=null) => { const steps = document.querySelectorAll(".steps:not(.is-hidden) .steps-marker") for (const step of steps) { - step.classList.remove("is-light", "is-danger"); + step.classList.remove("is-light", "is-danger", "is-dark"); step.parentElement.classList.remove("is-active", "is-dashed"); - step.querySelector("i").classList.remove("mdi-pencil", "mdi-check"); + step.querySelector("i").classList.remove("mdi-pencil", "mdi-check", "mdi-lock"); } diff --git a/www/sc_workbench.html b/www/sc_workbench.html index dd3089c6..df6e4fb4 100644 --- a/www/sc_workbench.html +++ b/www/sc_workbench.html @@ -1487,6 +1487,10 @@
+ + + + From d956c435863c26cefe170d7ce6e60ad2e0238f6f Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Thu, 8 Aug 2024 10:16:33 -0400 Subject: [PATCH 06/57] Resolving #854 --- www/js/classes/analysis-ui.js | 5 - www/js/classes/analysis.js | 37 ++++---- www/js/sc_workbench.js | 171 ++++++++++++++++++++++++++-------- www/sc_workbench.html | 21 ----- 4 files changed, 152 insertions(+), 82 deletions(-) diff --git a/www/js/classes/analysis-ui.js b/www/js/classes/analysis-ui.js index dff5f5d7..37277a09 100644 --- a/www/js/classes/analysis-ui.js +++ b/www/js/classes/analysis-ui.js @@ -22,11 +22,6 @@ class AnalysisUI { currentAnalysisElt = "#current-analysis" analysisSelect = "#analysis-select" newAnalysisOptionElt = `${this.analysisSelect} option[data-analysis-id='0']` - newAnalysisLabelContainer = "#new-analysis-label-c" - newAnalysisLabelElt = "#new-analysis-label" - btnNewAnalysisLabelSaveElt = "#btn-new-analysis-label-save" - btnNewAnalysisLabelCancelElt = "#btn-new-analysis-label-cancel" - duplicateLabelWarningElt = "#duplicate-label-warning" analysisPrimaryElt = "#analyses-primary" analysisPrimaryNotificationElt = "#analysis-primary-notification" analysisUnsavedElt = "#analyses-unsaved" diff --git a/www/js/classes/analysis.js b/www/js/classes/analysis.js index 846c9154..ecfc00c3 100644 --- a/www/js/classes/analysis.js +++ b/www/js/classes/analysis.js @@ -152,23 +152,29 @@ class Analysis { * @throws {Error} If the deletion is unsuccessful or an error occurs. */ async delete() { - const {data} = await axios.post("./cgi/delete_dataset_analysis.cgi", convertToFormData({ - session_id: this.userSessionId, - dataset_id: this.dataset.id, - analysis_id: this.id, - analysis_type: this.type - })); - if (!data.success || data.success < 1) { - const error = data.error || "Unknown error. Please contact gEAR support."; - throw new Error(error); - } + try { + const {data} = await axios.post("./cgi/delete_dataset_analysis.cgi", convertToFormData({ + session_id: this.userSessionId, + dataset_id: this.dataset.id, + analysis_id: this.id, + analysis_type: this.type + })); - // Trigger the selection of a 'New' analysis - document.querySelector(UI.newAnalysisOptionElt).setAttribute("selected", "selected"); - document.querySelector(UI.analysisSelect).dispatchEvent(new Event("change")); - await this.getSavedAnalysesList(this.dataset.id, 0); - resetStepperWithHrefs(UI.primaryFilterSection); + if (!data.success || data.success < 1) { + const error = data.error || "Unknown error. Please contact gEAR support."; + throw new Error(error); + } + + // Trigger the selection of a 'New' analysis + document.querySelector(UI.newAnalysisOptionElt).setAttribute("selected", "selected"); + document.querySelector(UI.analysisSelect).dispatchEvent(new Event("change")); + await this.getSavedAnalysesList(this.dataset.id, 0); + resetStepperWithHrefs(UI.primaryFilterSection); + + } catch (error) { + createToast(`Error deleting analysis: ${error.message}`); + } } async download() { @@ -688,7 +694,6 @@ class Analysis { document.querySelector(UI.analysisStatusInfoContainer).classList.remove("is-hidden"); document.querySelector(UI.btnDeleteSavedAnalysisElt).classList.remove("is-hidden"); document.querySelector(UI.btnMakePublicCopyElt).classList.remove("is-hidden"); - document.querySelector(UI.newAnalysisLabelContainer).classList.add("is-hidden"); await this.getSavedAnalysesList(this.dataset.id, this.id); } catch (error) { diff --git a/www/js/sc_workbench.js b/www/js/sc_workbench.js index eb6b4ff6..d2b9cb6e 100644 --- a/www/js/sc_workbench.js +++ b/www/js/sc_workbench.js @@ -317,7 +317,6 @@ const resetWorkbench = () => { elt.replaceChildren(); } */ - document.querySelector(UI.newAnalysisLabelElt).value = ''; } /** @@ -628,24 +627,140 @@ for (const button of document.querySelectorAll(UI.analysisDownloadElts)) { // Show the "rename" analysis label input when the button is clicked for (const button of document.querySelectorAll(UI.analysisRenameElts)) { button.addEventListener("click", (event) => { - document.querySelector(UI.newAnalysisLabelElt).value = currentAnalysis.label; - document.querySelector(UI.newAnalysisLabelContainer).classList.remove("is-hidden"); + + // remove existing popovers + const existingPopover = document.getElementById('rename-analysis-popover'); + if (existingPopover) { + existingPopover.remove(); + } + + // Create popover content + const popoverContent = document.createElement('article'); + popoverContent.id = 'rename-analysis-popover'; + popoverContent.classList.add("message", "is-dark"); + popoverContent.setAttribute("role", "tooltip"); + popoverContent.style.width = "500px"; + popoverContent.innerHTML = ` +
+

Rename collection

+
+
+

Please provide a new name for this analysis

+
+
+ +
+
+ +
+

+ +

+

+ +

+
+
+
+ `; + + // append element to DOM to get its dimensions + document.body.appendChild(popoverContent); + + document.getElementById("new-analysis-label").value = currentAnalysis.label; document.querySelector(UI.currentAnalysisElt).textContent = currentAnalysis.label; + + + const arrowElement = document.getElementById('arrow'); + + // Create popover (help from https://floating-ui.com/docs/tutorial) + computePosition(button, popoverContent, { + placement: 'bottom', + middleware: [ + flip(), // flip to top if there is not enough space on butotm + shift(), // shift the popover to the right if there is not enough space on the left + offset(5), // offset relative to the button + arrow({ element: arrowElement }) // add an arrow pointing to the button + ], + }).then(({ x, y, placement, middlewareData }) => { + console.log('Popover position:', x, y, placement, middlewareData); + // Position the popover + Object.assign(popoverContent.style, { + left: `${x}px`, + top: `${y}px`, + }); + // Accessing the data + const { x: arrowX, y: arrowY } = middlewareData.arrow; + + // Position the arrow relative to the popover + const staticSide = { + top: 'bottom', + right: 'left', + bottom: 'top', + left: 'right', + }[placement.split('-')[0]]; + + // Set the arrow position + Object.assign(arrowElement.style, { + left: arrowX != null ? `${arrowX}px` : '', + top: arrowY != null ? `${arrowY}px` : '', + right: '', + bottom: '', + [staticSide]: '-4px', + }); + }); + + document.getElementById("new-analysis-label").addEventListener("keyup", (event) => { + // Update the new analysis label if it is not a duplicate + if (analysisLabels.has(event.target.value.trim())) { + if (event.target.value.trim() !== currentLabel) { + event.target.classList.add("duplicate"); + document.getElementById("confirm-analysis-rename").disabled = true; + document.getElementById("duplicate-label-warning").classList.remove("is-hidden"); + } + return; + } + + document.getElementById("duplicate-label-warning").classList.add("is-hidden"); + event.target.classList.remove("duplicate"); + document.getElementById("confirm-analysis-rename").disabled = false; + }); + + document.getElementById("new-analysis-label").addEventListener("focus", (event) => { + // Reset the new analysis label + currentLabel = event.target.value.trim(); + }); + + document.getElementById("cancel-analysis-rename").addEventListener("click", async (event) => { + // Reset the label to the current analysis label + document.getElementById("new-analysis-label").value = currentAnalysis.label; + popoverContent.remove(); + }); + + + document.getElementById("confirm-analysis-rename").addEventListener("click", async (event) => { + event.target.classList.add("is-loading"); + currentAnalysis.label = document.getElementById("new-analysis-label").value; + + try { + // Save the new label to the current analysis + await currentAnalysis.save(); + } catch (error) { + // pass - handled in the save method + } finally { + event.target.classList.remove("is-loading"); + popoverContent.remove(); + } + }); + }); } -document.querySelector(UI.btnNewAnalysisLabelSaveElt).addEventListener("click", async (event) => { - // Save the new label to the current analysis - currentAnalysis.label = document.querySelector(UI.newAnalysisLabelElt).value; - await currentAnalysis.save(); - document.querySelector(UI.newAnalysisLabelContainer).classList.add("is-hidden"); -}); -document.querySelector(UI.btnNewAnalysisLabelCancelElt).addEventListener("click", async (event) => { - // Reset the label to the current analysis label - document.querySelector(UI.newAnalysisLabelElt).value = currentAnalysis.label; - document.querySelector(UI.newAnalysisLabelContainer).classList.add("is-hidden"); -}); + + /** * Creates a confirmation popover for deleting an analysis. @@ -736,10 +851,9 @@ for (const button of deleteButtons) { try { // Delete the current analysis - currentAnalysis.delete(); + await currentAnalysis.delete(); } catch (error) { - logErrorInConsole(error); - createToast("Failed to delete dataset"); + // pass - handled in the delete method } finally { event.target.classList.remove("is-loading"); popoverContent.remove(); @@ -800,8 +914,6 @@ document.querySelector(UI.analysisSelect).addEventListener("change", async (even } createToast("Loading stored analysis", "is-info"); - document.querySelector(UI.newAnalysisLabelContainer).classList.add("is-hidden"); - resetWorkbench(); const selectedOption = event.target.selectedOptions[0]; @@ -842,27 +954,6 @@ document.querySelector(UI.analysisSelect).addEventListener("change", async (even }); -document.querySelector(UI.newAnalysisLabelElt).addEventListener("keyup", (event) => { - // Update the new analysis label if it is not a duplicate - if (analysisLabels.has(event.target.value.trim())) { - if (event.target.value.trim() !== currentLabel) { - event.target.classList.add("duplicate"); - document.querySelector(UI.btnNewAnalysisLabelSaveElt).disabled = true; - document.querySelector(UI.duplicateLabelWarningElt).classList.remove("is-hidden"); - } - return; - } - - document.querySelector(UI.duplicateLabelWarningElt).classList.add("is-hidden"); - event.target.classList.remove("duplicate"); - document.querySelector(UI.btnNewAnalysisLabelSaveElt).disabled = false; -}); - -document.querySelector(UI.newAnalysisLabelElt).addEventListener("focus", (event) => { - // Reset the new analysis label - currentLabel = event.target.value.trim(); -}); - // Labeled tSNE document.querySelector(UI.btnLabeledTsneRunElt).addEventListener("click", async (event) => { diff --git a/www/sc_workbench.html b/www/sc_workbench.html index df6e4fb4..0c805652 100644 --- a/www/sc_workbench.html +++ b/www/sc_workbench.html @@ -185,27 +185,6 @@
Initial composition plots
-
diff --git a/www/plugins/deafness_gene_annotation/expression.js b/www/plugins/deafness_gene_annotation/expression.js index 3eccbec9..d58c7771 100644 --- a/www/plugins/deafness_gene_annotation/expression.js +++ b/www/plugins/deafness_gene_annotation/expression.js @@ -6,13 +6,20 @@ let mgi_data; function add_dga_button_listeners() { document.querySelectorAll('.btn-dm').forEach(item => { item.addEventListener('click', event => { - const dm = item.closest('.dropdown'); + let dm = item.closest('.dropdown'); if (dm.classList.contains('is-active')) { dm.classList.remove('is-active'); } else { dm.classList.add('is-active'); } + + document.querySelectorAll('.btn-dm').forEach(btn => { + if (btn !== item) { + dm = btn.closest('.dropdown'); + dm.classList.remove('is-active'); + } + }); }); }); } @@ -60,40 +67,45 @@ function deafness_plugin_gene_change() { document.getElementById("img-deafness-gene-mouse").src = "./img/icons/org-1-dark-64.svg"; document.getElementById("btn-deafness-gene-mouse").disabled = false; } - - return; - - - if (impc_data.hasOwnProperty(gene_symbol)) { - $('#deafness_gene_mouse').attr('data-title', impc_data[gene_symbol]['on_hover']); - mouse_deafness_links = mouse_deafness_links.concat(impc_data[gene_symbol]['links']); - } + if (omim_data.hasOwnProperty(gene_symbol)) { + for (const phenotype of omim_data[gene_symbol]['phenotypes']) { + const span = phenotype_template.content.cloneNode(true); + span.querySelector('span').innerHTML = phenotype; + document.getElementById('dm-deafness-gene-human-phenotypes').appendChild(span); + } - if (mouse_deafness_links.length > 0) { - $("button#deafness_gene_mouse").attr("disabled", false); - $('#deafness_gene_mouse img').attr('src', './img/icons/org-1-dark-64.svg'); - var links_tmpl = $.templates("#tmpl_deafness_resource_links"); - var links_html = links_tmpl.render(mouse_deafness_links); - $("#deafness_popover_links").html(links_html); + for (const link of omim_data[gene_symbol]['links']) { + const a = link_template.content.cloneNode(true); + a.querySelector('a').innerHTML = link['label']; + a.querySelector('a').href = link['url']; + document.getElementById('dm-deafness-gene-human-links').appendChild(a); + } - $('#deafness_gene_mouse').attr("data-popover", $("#deafness_popover_c").html()) + document.getElementById("img-deafness-gene-human").src = "./img/icons/org-2-dark-64.svg"; + document.getElementById("btn-deafness-gene-human").disabled = false; } - // this is here after mouse so that the phenotypes don't carry over to other organisms - $('#deafness_popover_phenotypes').html('No data available'); + if (hl_data.hasOwnProperty(gene_symbol)) { + const locus = hl_data[gene_symbol]['locus']; + const span = phenotype_template.content.cloneNode(true); + span.querySelector('span').innerHTML = locus; + document.getElementById('dm-deafness-gene-human-putative-loci').appendChild(span); - if (omim_data.hasOwnProperty(gene_symbol)) { - $("button#deafness_gene_human").attr("disabled", false); - $('#deafness_gene_human img').attr('src', './img/icons/org-2-dark-64.svg'); - $('#deafness_gene_human').attr('data-title', - omim_data[gene_symbol]['phenotypes'].join(' - ')); + for (const link of hl_data[gene_symbol]['links']) { + const a = link_template.content.cloneNode(true); + a.querySelector('a').innerHTML = link['label']; + a.querySelector('a').href = link['url']; + document.getElementById('dm-deafness-gene-human-putative-links').appendChild(a); + } - var links_tmpl = $.templates("#tmpl_deafness_resource_links"); - var links_html = links_tmpl.render(omim_data[gene_symbol]['links']); - $("#deafness_popover_links").html(links_html); - $('#deafness_gene_human').attr("data-popover", $("#deafness_popover_c").html()) + document.getElementById("img-deafness-gene-human-putative").src = "./img/icons/org-2-dark-64.svg"; + document.getElementById("btn-deafness-gene-human-putative").disabled = false; } + + return; + + if (hl_data.hasOwnProperty(gene_symbol)) { $("button#deafness_gene_human_putative").attr("disabled", false); @@ -106,21 +118,6 @@ function deafness_plugin_gene_change() { $("#deafness_popover_links").html(links_html); $('#deafness_gene_human_putative').attr("data-popover", $("#deafness_popover_c").html()) } - - $('button.icon-deafness-gene').each(function() { - $(this).popover("dispose").popover({ - content : $(this).attr("data-popover"), - placement : 'auto', - title : 'Deafness gene info', - trigger: 'focus', - html: true - }).tooltip("dispose").tooltip({ - placement : 'top', - title : $(this).attr("data-title") - }).on('show.bs.popover', function() { - $(this).tooltip('hide') - }) - }); } From 4f2118da9bca45c2b36acbb65638c44d043336b2 Mon Sep 17 00:00:00 2001 From: Joshua Orvis Date: Fri, 9 Aug 2024 10:38:37 -0500 Subject: [PATCH 19/57] Fixed issue where changing genes wasn't clearing all phenotype/links --- www/plugins/deafness_gene_annotation/expression.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/www/plugins/deafness_gene_annotation/expression.js b/www/plugins/deafness_gene_annotation/expression.js index d58c7771..82d7e98a 100644 --- a/www/plugins/deafness_gene_annotation/expression.js +++ b/www/plugins/deafness_gene_annotation/expression.js @@ -9,8 +9,10 @@ function add_dga_button_listeners() { let dm = item.closest('.dropdown'); if (dm.classList.contains('is-active')) { + console.log("removing active class"); dm.classList.remove('is-active'); } else { + console.log("adding active class"); dm.classList.add('is-active'); } @@ -44,8 +46,13 @@ function deafness_plugin_gene_change() { document.getElementById("img-deafness-gene-human-putative").src = "./img/icons/org-2-unknown-outline-64.svg"; // Clear any existing lists - document.querySelector(".dm-deafness-phenotypes").innerHTML = ""; - document.querySelector(".dm-deafness-links").innerHTML = ""; + for (const list of document.querySelectorAll(".dm-deafness-phenotypes")) { + list.innerHTML = ""; + } + + for (const list of document.querySelectorAll(".dm-deafness-links")) { + list.innerHTML = ""; + } const phenotype_template = document.getElementById('tmpl-deafness-phenotype'); const link_template = document.getElementById('tmpl-deafness-resource-link'); From 94c7be67d1677f207392063fc6c859c2be959496 Mon Sep 17 00:00:00 2001 From: Joshua Orvis Date: Fri, 9 Aug 2024 10:39:23 -0500 Subject: [PATCH 20/57] Removed older/commented code block --- .../deafness_gene_annotation/expression.js | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/www/plugins/deafness_gene_annotation/expression.js b/www/plugins/deafness_gene_annotation/expression.js index 82d7e98a..acd6ad49 100644 --- a/www/plugins/deafness_gene_annotation/expression.js +++ b/www/plugins/deafness_gene_annotation/expression.js @@ -109,22 +109,6 @@ function deafness_plugin_gene_change() { document.getElementById("img-deafness-gene-human-putative").src = "./img/icons/org-2-dark-64.svg"; document.getElementById("btn-deafness-gene-human-putative").disabled = false; } - - return; - - - - if (hl_data.hasOwnProperty(gene_symbol)) { - $("button#deafness_gene_human_putative").attr("disabled", false); - $('#deafness_gene_human_putative img').attr('src', './img/icons/org-2-unknown-dark-64.svg'); - $('#deafness_gene_human_putative').attr('data-title', - hl_data[gene_symbol]['locus']); - - var links_tmpl = $.templates("#tmpl_deafness_resource_links"); - var links_html = links_tmpl.render(hl_data[gene_symbol]['links']); - $("#deafness_popover_links").html(links_html); - $('#deafness_gene_human_putative').attr("data-popover", $("#deafness_popover_c").html()) - } } From 8421ed8ddb1a718ec0a0a777e65e2b23e6f3d1e3 Mon Sep 17 00:00:00 2001 From: Joshua Orvis Date: Fri, 9 Aug 2024 10:48:08 -0500 Subject: [PATCH 21/57] Fixed issue where click events were being re-registered when the gene changed --- www/plugins/deafness_gene_annotation/expression.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/www/plugins/deafness_gene_annotation/expression.js b/www/plugins/deafness_gene_annotation/expression.js index acd6ad49..88392816 100644 --- a/www/plugins/deafness_gene_annotation/expression.js +++ b/www/plugins/deafness_gene_annotation/expression.js @@ -9,10 +9,8 @@ function add_dga_button_listeners() { let dm = item.closest('.dropdown'); if (dm.classList.contains('is-active')) { - console.log("removing active class"); dm.classList.remove('is-active'); } else { - console.log("adding active class"); dm.classList.add('is-active'); } @@ -202,4 +200,4 @@ fetch("./plugins/deafness_gene_annotation/mgi_data.json") }); geneChangeCallbacks.push(deafness_plugin_gene_change); -geneChangeCallbacks.push(add_dga_button_listeners); +add_dga_button_listeners(); From 3c2a2e7982068ff9ee01ead2eaed8cf5abb84cc0 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Fri, 9 Aug 2024 13:41:53 -0400 Subject: [PATCH 22/57] moving adata into try block to keep consistent with other similar calls --- www/api/resources/projectr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/api/resources/projectr.py b/www/api/resources/projectr.py index 4a8401dd..99f79707 100644 --- a/www/api/resources/projectr.py +++ b/www/api/resources/projectr.py @@ -296,6 +296,8 @@ def projectr_callback(dataset_id, genecart_id, projection_id, session_id, scope, # NOTE Currently no analyses are supported yet. try: ana = geardb.get_analysis(None, dataset_id, session_id) + # Using adata with "backed" mode does not work with volcano plot + adata = ana.get_adata(backed=True) except Exception as e: print(str(e), file=fh) return { @@ -303,8 +305,6 @@ def projectr_callback(dataset_id, genecart_id, projection_id, session_id, scope, , 'message': str(e) } - # Using adata with "backed" mode does not work with volcano plot - adata = ana.get_adata(backed=True) # If dataset genes have duplicated index names, we need to rename them to avoid errors # in collecting rownames in projectR (which gives invalid output) From e050be8dfc6c78b59b3b89db411b974f64b08cb6 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Fri, 9 Aug 2024 14:44:29 -0400 Subject: [PATCH 23/57] Obfuscating "no analysis" errors --- www/api/resources/multigene_dash_data.py | 16 ++++++++++++---- www/api/resources/plotly_data.py | 10 +++++++++- www/api/resources/projectr.py | 16 +++++++++++++--- www/api/resources/tsne_data.py | 12 ++++++++++-- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/www/api/resources/multigene_dash_data.py b/www/api/resources/multigene_dash_data.py index 56acdda4..b4586b4e 100644 --- a/www/api/resources/multigene_dash_data.py +++ b/www/api/resources/multigene_dash_data.py @@ -142,14 +142,22 @@ def post(self, dataset_id): try: ana = geardb.get_analysis(analysis, dataset_id, session_id) - # Using adata with "backed" mode does not work with volcano plot - adata = ana.get_adata(backed=False) except Exception as e: import traceback traceback.print_exc() return { - 'success': -1, - 'message': str(e), + "success": -1, + "message": "Could not retrieve analysis." + } + + try: + adata = ana.get_adata(backed=True) + except Exception as e: + import traceback + traceback.print_exc() + return { + "success": -1, + "message": "Could not retrieve AnnData object." } adata.obs = order_by_time_point(adata.obs) diff --git a/www/api/resources/plotly_data.py b/www/api/resources/plotly_data.py index 41c21376..456fbbcd 100644 --- a/www/api/resources/plotly_data.py +++ b/www/api/resources/plotly_data.py @@ -119,12 +119,20 @@ def post(self, dataset_id): try: ana = geardb.get_analysis(analysis, dataset_id, session_id) + except Exception as e: + import traceback + traceback.print_exc() + return_dict["success"] = -1 + return_dict["message"] = "Could not retrieve analysis." + return return_dict + + try: adata = ana.get_adata(backed=True) except Exception as e: import traceback traceback.print_exc() return_dict["success"] = -1 - return_dict["message"] = str(e) + return_dict["message"] = "Could not retrieve AnnData." return return_dict if projection_id: diff --git a/www/api/resources/projectr.py b/www/api/resources/projectr.py index 99f79707..8b34cf2c 100644 --- a/www/api/resources/projectr.py +++ b/www/api/resources/projectr.py @@ -296,13 +296,23 @@ def projectr_callback(dataset_id, genecart_id, projection_id, session_id, scope, # NOTE Currently no analyses are supported yet. try: ana = geardb.get_analysis(None, dataset_id, session_id) + except Exception as e: + import traceback + traceback.print_exc() + return { + "success": -1, + "message": "Could not retrieve analysis." + } + + try: # Using adata with "backed" mode does not work with volcano plot adata = ana.get_adata(backed=True) except Exception as e: - print(str(e), file=fh) + import traceback + traceback.print_exc() return { - 'success': -1 - , 'message': str(e) + "success": -1, + "message": "Could not retrieve AnnData object." } diff --git a/www/api/resources/tsne_data.py b/www/api/resources/tsne_data.py index 4e7f0d92..b9a64aed 100644 --- a/www/api/resources/tsne_data.py +++ b/www/api/resources/tsne_data.py @@ -211,15 +211,23 @@ def post(self, dataset_id): try: ana = geardb.get_analysis(analysis, dataset_id, session_id) - adata = ana.get_adata(backed=True) except Exception as e: import traceback traceback.print_exc() return { "success": -1, - "message": str(e) + "message": "Could not retrieve analysis." } + try: + adata = ana.get_adata(backed=True) + except Exception as e: + import traceback + traceback.print_exc() + return { + "success": -1, + "message": "Could not retrieve AnnData object." + } if projection_id: try: From 1551df6d4cdf26736b4483c9df77af72ba7d27bc Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Fri, 9 Aug 2024 15:34:53 -0400 Subject: [PATCH 24/57] fixing bad Organism key --- lib/gear/orthology.py | 6 +++++- lib/geardb.py | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/gear/orthology.py b/lib/gear/orthology.py index 02ba5d99..34e1daff 100644 --- a/lib/gear/orthology.py +++ b/lib/gear/orthology.py @@ -109,7 +109,11 @@ def get_organism_name_by_id(organism_id: str): Returns: str: The organism name corresponding to the given organism ID. """ - return filter_organism_by_id(organism_id)["name"] + organism = filter_organism_by_id(organism_id) + if organism is not None: + return organism["label"] + else: + return "" def create_orthology_df(orthomap_file: Path): """ diff --git a/lib/geardb.py b/lib/geardb.py index 28f337d6..8724a868 100644 --- a/lib/geardb.py +++ b/lib/geardb.py @@ -910,8 +910,8 @@ def __repr__(self): return json.dumps(self.__dict__) class OrganismCollection: - def __init__(self, organisms=None): - self.organisms = [] if organisms is None else organisms + def __init__(self, organisms=[]): + self.organisms = organisms def __repr__(self): return json.dumps(self.__dict__) @@ -944,6 +944,7 @@ def get_all(self): self.organisms.append(org) cursor.close() + conn.close() return self.organisms From 62052c2de74e283dcfad1832d07885576508d37a Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Fri, 9 Aug 2024 15:59:16 -0400 Subject: [PATCH 25/57] (#623) Refactor geardb.py to check for h5ad file existence before proceeding --- lib/geardb.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/geardb.py b/lib/geardb.py index 8724a868..cf15e63e 100644 --- a/lib/geardb.py +++ b/lib/geardb.py @@ -105,6 +105,11 @@ def get_analysis(analysis, dataset_id, session_id): ana.type = analysis['type'] else: ana.discover_type() + + # Check that the h5ad file exists + if not os.path.exists(ana.dataset_path()): + raise FileNotFoundError("No h5 file found for the passed in analysis") + else: ds = Dataset(id=dataset_id, has_h5ad=1) h5_path = ds.get_file_path() From 47beec93e023f26c8c64c5af32a3733275df0681 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 09:54:26 -0400 Subject: [PATCH 26/57] Refactor update_share_id.cgi to use more relative paths --- www/cgi/update_share_id.cgi | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/www/cgi/update_share_id.cgi b/www/cgi/update_share_id.cgi index d1d79c80..4e1ea9fd 100755 --- a/www/cgi/update_share_id.cgi +++ b/www/cgi/update_share_id.cgi @@ -10,6 +10,13 @@ lib_path = os.path.abspath(os.path.join('..', '..', 'lib')) sys.path.append(lib_path) import geardb +from pathlib import Path +abs_path_www = Path(__file__).resolve().parents[1] # web-root dir +CARTS_BASE_DIR = abs_path_www.joinpath("carts") +BY_DATASET_DIR = abs_path_www.joinpath("projections", "by_dataset") +BY_GENECART_DIR = abs_path_www.joinpath("projections", "by_genecart") + + def main(): cnx = geardb.Connection() print('Content-Type: application/json\n\n') @@ -115,7 +122,7 @@ def main(): # rename carts if gene_cart.gctype == "weighted": - os.chdir("/var/www/carts") + os.chdir(str(CARTS_BASE_DIR)) for filename in os.listdir("."): if not share_id in filename: @@ -131,7 +138,7 @@ def main(): # rename projections # the "by_dataset" directory only needs projections.json files updated - os.chdir("/var/www/projections/by_dataset") + os.chdir(str(BY_DATASET_DIR)) for root, dirs, files in os.walk("."): for file in files: if not file.endswith("projections.json"): @@ -145,7 +152,7 @@ def main(): break # the "by_genecart" directory needs both directory names updated - os.chdir("/var/www/projections/by_genecart") + os.chdir(str(BY_GENECART_DIR)) for root, dirs, files in os.walk("."): for dirname in dirs: if not share_id in dirname: From 56a91bcb621ea5d3b01d400e6291f9d0f6ac610d Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 10:03:50 -0400 Subject: [PATCH 27/57] (#768) Resolved duplication bug by removing extra "select collection" calls --- www/js/dataset_explorer.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/www/js/dataset_explorer.js b/www/js/dataset_explorer.js index 336ac71a..aa9e4bc2 100644 --- a/www/js/dataset_explorer.js +++ b/www/js/dataset_explorer.js @@ -578,11 +578,8 @@ const addModalEventListeners = () => { createToast("Display added to collection", "is-success"); - const curr_share_id = selected_dc_share_id; - // Update the layout arrangement views await updateDatasetCollections(); - selectDatasetCollection(curr_share_id); // performs DatasetCollectionSelectorCallback when label is set } catch (error) { logErrorInConsole(error); @@ -615,12 +612,8 @@ const addModalEventListeners = () => { createToast("Display removed from collection", "is-success"); - const curr_share_id = selected_dc_share_id; - // Update the layout arrangement views await updateDatasetCollections(); - selectDatasetCollection(curr_share_id); // performs DatasetCollectionSelectorCallback when label is set - } catch (error) { logErrorInConsole(error); createToast("Failed to remove dataset from collection"); @@ -2196,7 +2189,6 @@ const renderDisplaysModalDisplays = async (displays, collection, displayElt, dat const displayCount = displayElement.querySelector('.js-collection-display-count'); if (collection?.members) { - console.log(collection.members); const displayCountValue = collection.members.filter((member) => JSON.parse(member).display_id === displayId).length displayCount.textContent = displayCountValue; } else { From f00bf793828eaa2b894456f6662b94fefb866f75 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 10:47:17 -0400 Subject: [PATCH 28/57] (#855) Resolving and closing --- www/js/dataset_curator.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/www/js/dataset_curator.js b/www/js/dataset_curator.js index 88784845..8e0f926b 100644 --- a/www/js/dataset_curator.js +++ b/www/js/dataset_curator.js @@ -1121,15 +1121,16 @@ const setupPlotlyOptions = async () => { const difference = (arr1, arr2) => arr1.filter(x => !arr2.includes(x)) const continuousColumns = difference(allColumns, catColumns); - // class name, list of columns, add expression, default category - const xColumns = ["bar", "violin"].includes(plotType) ? catColumns : allColumns; const xUseRaw = ["bar", "violin"].includes(plotType) ? false : true; const yColumns = ["bar", "violin"].includes(plotType) ? continuousColumns : allColumns; + const colorColumns = ["bar", "line", "violin"].includes(plotType) ? catColumns : allColumns; + const colorUseRaw = ["bar", "line", "violin"].includes(plotType) ? false : true; + // Arguments - class name, list of columns, add expression, default category updateSeriesOptions("js-plotly-x-axis", xColumns, xUseRaw); updateSeriesOptions("js-plotly-y-axis", yColumns, true, "raw_value"); - updateSeriesOptions("js-plotly-color", allColumns, true); + updateSeriesOptions("js-plotly-color", colorColumns, colorUseRaw); updateSeriesOptions("js-plotly-label", allColumns, true); updateSeriesOptions("js-plotly-facet-row", catColumns, false); updateSeriesOptions("js-plotly-facet-col", catColumns, false); From dbf77e3f4f7b9ed9b97803d05afbc0c360604179 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 11:05:57 -0400 Subject: [PATCH 29/57] Updating code to work for #666 --- services/projectr/install_bioc.sh | 3 +-- services/projectr/main.py | 9 ++++++++- services/projectr/requirements.txt | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/services/projectr/install_bioc.sh b/services/projectr/install_bioc.sh index 04c72436..acd17f9c 100755 --- a/services/projectr/install_bioc.sh +++ b/services/projectr/install_bioc.sh @@ -5,8 +5,7 @@ Rver="${Rmaj}.3.1" current_dir=$(pwd) -# Install and build R (Using 'apt-get install' on Ubuntu Trusty installs version 3.0.2 of R) -curl http://lib.stat.cmu.edu/R/CRAN/src/base/${Rmaj}/${Rver}.tar.gz | tar -C /opt -zx +curl -s -L http://lib.stat.cmu.edu/R/CRAN/src/base/${Rmaj}/${Rver}.tar.gz | tar xzv -C /opt cd /opt/${Rver} /opt/${Rver}/configure --with-readline=no --enable-R-shlib --enable-BLAS-shlib --with-x=no || exit 1 make || exit 1 diff --git a/services/projectr/main.py b/services/projectr/main.py index 5d77ec81..7f78b6ad 100644 --- a/services/projectr/main.py +++ b/services/projectr/main.py @@ -1,5 +1,6 @@ import os, sys import pandas as pd +from io import StringIO from flask import Flask, abort, jsonify, request cloud_logging = False @@ -38,7 +39,9 @@ def do_binary_projection(target_df, loading_df): """Perform projection based on the number of genes that were expressed in the cell or observation.""" # Only applies with unweighted gene carts. tp_target_series = target_df.astype(bool).sum(axis=0).transpose() - return pd.DataFrame(data=tp_target_series, columns=loading_df.columns, index=tp_target_series.index) + # Need to convert data to list of lists to create a DataFrame. + # https://stackoverflow.com/questions/70854450/pandas-dataframe-shape-of-passed-values-is-5-1-indices-imply-5-2 + return pd.DataFrame(data=[tp_target_series], columns=loading_df.columns, index=tp_target_series.index) def do_pca_projection(target_df, loading_df): """Perform projection of PCA loadings.""" @@ -66,6 +69,10 @@ def index(): write_entry("projectr", "INFO", "Genecart ID: {}".format(genecart_id)) + # pd.read_json gives a FutureWarning, and suggest to wrap the json in StringIO. + target = StringIO(target) + loadings = StringIO(loadings) + target_df = pd.read_json(target, orient="split") loading_df = pd.read_json(loadings, orient="split") diff --git a/services/projectr/requirements.txt b/services/projectr/requirements.txt index b8bb156b..6cc46019 100644 --- a/services/projectr/requirements.txt +++ b/services/projectr/requirements.txt @@ -1,5 +1,5 @@ Flask==3.0.0 gunicorn==20.1.0 rpy2==3.5.1 # 3.5.2 and up gives errors with rpy2py and py2rpy -pandas==1.4.1 +pandas==2.2.1 google-cloud-logging \ No newline at end of file From a39959f33491767fc594c1774bfb653f564e92c6 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 13:56:51 -0400 Subject: [PATCH 30/57] #865 Resolved --- lib/gear/orthology.py | 8 +++++--- lib/geardb.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/gear/orthology.py b/lib/gear/orthology.py index 34e1daff..0f720691 100644 --- a/lib/gear/orthology.py +++ b/lib/gear/orthology.py @@ -98,7 +98,8 @@ def filter_organism_by_id(organism_id: str): Returns: dict: The organism dictionary corresponding to the given organism ID. """ - return next((item for item in organisms if item["id"] == organism_id), None) + + return next((item for item in organisms if item.id == organism_id), None) def get_organism_name_by_id(organism_id: str): """Get the organism name corresponding to the given organism ID. @@ -111,7 +112,7 @@ def get_organism_name_by_id(organism_id: str): """ organism = filter_organism_by_id(organism_id) if organism is not None: - return organism["label"] + return organism.label else: return "" @@ -153,7 +154,8 @@ def map_dataframe_genes(orig_df: pd.DataFrame, orthomap_file: Path): def get_best_match(id1): # Get the best match for the id2 gene symbol - sorted_by_best_match = orthomap_df[orthomap_df["id1"] == id1].sort_values("algorithms_match_count", ascending=False) + best_match_for_id = orthomap_df[orthomap_df["id1"] == id1] + sorted_by_best_match = best_match_for_id.sort_values(by="algorithms_match_count", ascending=False) # If no match, return the original id1 if sorted_by_best_match.empty: return id1 diff --git a/lib/geardb.py b/lib/geardb.py index cf15e63e..c638d2db 100644 --- a/lib/geardb.py +++ b/lib/geardb.py @@ -914,9 +914,9 @@ def __init__(self, id=None, label=None, genus=None, species=None, strain=None, t def __repr__(self): return json.dumps(self.__dict__) +@dataclass class OrganismCollection: - def __init__(self, organisms=[]): - self.organisms = organisms + organisms: List[Organism] = field(default_factory=list) def __repr__(self): return json.dumps(self.__dict__) From 986661a0738e5228863f93233016dc0146b1643e Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 15:21:47 -0400 Subject: [PATCH 31/57] (#666) binary projection should work on multiple binary weights instead of a single unweighted list --- services/projectr/main.py | 20 +++++++++++++++----- www/api/resources/projectr.py | 4 ++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/services/projectr/main.py b/services/projectr/main.py index 7f78b6ad..8114bbcd 100644 --- a/services/projectr/main.py +++ b/services/projectr/main.py @@ -37,11 +37,21 @@ def write_entry(logger_name, severity, message): def do_binary_projection(target_df, loading_df): """Perform projection based on the number of genes that were expressed in the cell or observation.""" - # Only applies with unweighted gene carts. - tp_target_series = target_df.astype(bool).sum(axis=0).transpose() - # Need to convert data to list of lists to create a DataFrame. - # https://stackoverflow.com/questions/70854450/pandas-dataframe-shape-of-passed-values-is-5-1-indices-imply-5-2 - return pd.DataFrame(data=[tp_target_series], columns=loading_df.columns, index=tp_target_series.index) + # Only applies with unweighted gene carts, or weighted carts with binary values. + + # for each loading pattern, count the number of genes that are expressed in the target + # and return the count as the pattern weight. + binary_target_df = pd.DataFrame() + for pattern in loading_df.columns: + # Count the number of genes that are 1 in the loading_df + good_loading_genes_mask = loading_df[pattern].astype(bool) + good_loading_genes = loading_df.index[good_loading_genes_mask] + + # Count the number of those genes that are 1 (expressed) in the target_df. + good_genes = target_df.loc[good_loading_genes].astype(bool).sum(axis=0).transpose() + binary_target_df[pattern] = good_genes + return binary_target_df + def do_pca_projection(target_df, loading_df): """Perform projection of PCA loadings.""" diff --git a/www/api/resources/projectr.py b/www/api/resources/projectr.py index 8b34cf2c..d0a1a05c 100644 --- a/www/api/resources/projectr.py +++ b/www/api/resources/projectr.py @@ -479,6 +479,9 @@ def projectr_callback(dataset_id, genecart_id, projection_id, session_id, scope, else: raise ValueError("Algorithm {} is not supported".format(algorithm)) except Exception as e: + # clear lock file + remove_lock_file(lock_fh, lockfile) + print(str(e), file=sys.stderr) return { 'success': -1 @@ -488,6 +491,7 @@ def projectr_callback(dataset_id, genecart_id, projection_id, session_id, scope, , "num_dataset_genes": num_target_genes } + # Have had cases where the column names are x1, x2, x3, etc. so load in the original pattern names projection_patterns_df = projection_patterns_df.set_axis(loading_df.columns, axis="columns") From ce2076a79376097f75058effbd7e1806afab22d0 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 19:00:22 -0400 Subject: [PATCH 32/57] Fixing UnboundError --- www/api/resources/projectr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/www/api/resources/projectr.py b/www/api/resources/projectr.py index d0a1a05c..0440923b 100644 --- a/www/api/resources/projectr.py +++ b/www/api/resources/projectr.py @@ -662,6 +662,7 @@ def post(self, dataset_id): dataset_genes = config.get('num_dataset_genes', -1) break + message = "" if common_genes: message = "Found {} common genes between the target dataset ({} genes) and the pattern file ({} genes).".format(common_genes, dataset_genes, genecart_genes) From 848acc8aef85f4b3744737b687c75e377cbdf289 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 19:18:02 -0400 Subject: [PATCH 33/57] Fixing bug where facets are requested but category is not in the 'order' dictionary --- lib/gear/plotting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gear/plotting.py b/lib/gear/plotting.py index f3bab73f..eb099073 100644 --- a/lib/gear/plotting.py +++ b/lib/gear/plotting.py @@ -468,10 +468,10 @@ def generate_plot(df, x=None, y=None, z=None, facet_row=None, facet_col=None, # TODO: put in function # Map indexes for subplot ordering. Indexes start at 1 since plotting rows/cols start at 1 - facet_row_groups = category_orders[facet_row] if facet_row and facet_row in category_orders else [] + facet_row_groups = category_orders[facet_row] if facet_row and facet_row in category_orders else df[facet_row].unique() facet_row_indexes = {group: idx for idx, group in enumerate(facet_row_groups, start=1)} num_rows = len(facet_row_groups) if facet_row else 1 - facet_col_groups = category_orders[facet_col] if facet_col and facet_col in category_orders else [] + facet_col_groups = category_orders[facet_col] if facet_col and facet_col in category_orders else df[facet_col].unique() facet_col_indexes = {group: idx for idx, group in enumerate(facet_col_groups, start=1)} num_cols = len(facet_col_groups) if facet_col else 1 From 9c6ad43d6e49400b1e37a42280a25b0ba8d0727b Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 19:20:34 -0400 Subject: [PATCH 34/57] Refactor generate_plot function to handle facet_row and facet_col groups properly --- lib/gear/plotting.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/gear/plotting.py b/lib/gear/plotting.py index eb099073..07d08947 100644 --- a/lib/gear/plotting.py +++ b/lib/gear/plotting.py @@ -468,10 +468,17 @@ def generate_plot(df, x=None, y=None, z=None, facet_row=None, facet_col=None, # TODO: put in function # Map indexes for subplot ordering. Indexes start at 1 since plotting rows/cols start at 1 - facet_row_groups = category_orders[facet_row] if facet_row and facet_row in category_orders else df[facet_row].unique() + facet_row_groups = [] + facet_col_groups = [] + + if facet_row: + facet_row_groups = category_orders[facet_row] if facet_row in category_orders else df[facet_row].unique() + + if facet_col: + facet_col_groups = category_orders[facet_col] if facet_col in category_orders else df[facet_col].unique() + facet_row_indexes = {group: idx for idx, group in enumerate(facet_row_groups, start=1)} num_rows = len(facet_row_groups) if facet_row else 1 - facet_col_groups = category_orders[facet_col] if facet_col and facet_col in category_orders else df[facet_col].unique() facet_col_indexes = {group: idx for idx, group in enumerate(facet_col_groups, start=1)} num_cols = len(facet_col_groups) if facet_col else 1 From 11e4f2ac5f780c6d1d21e4ff6de937fdcd6cc6bb Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 19:23:30 -0400 Subject: [PATCH 35/57] Refactor generate_plot function to handle facet_row and facet_col groups properly --- lib/gear/plotting.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/gear/plotting.py b/lib/gear/plotting.py index 07d08947..ad0481f6 100644 --- a/lib/gear/plotting.py +++ b/lib/gear/plotting.py @@ -470,12 +470,16 @@ def generate_plot(df, x=None, y=None, z=None, facet_row=None, facet_col=None, # Map indexes for subplot ordering. Indexes start at 1 since plotting rows/cols start at 1 facet_row_groups = [] facet_col_groups = [] + row_titles = None + column_titles = None if facet_row: facet_row_groups = category_orders[facet_row] if facet_row in category_orders else df[facet_row].unique() + row_titles = facet_row_groups if facet_col: facet_col_groups = category_orders[facet_col] if facet_col in category_orders else df[facet_col].unique() + column_titles = facet_col_groups facet_row_indexes = {group: idx for idx, group in enumerate(facet_row_groups, start=1)} num_rows = len(facet_row_groups) if facet_row else 1 @@ -485,8 +489,8 @@ def generate_plot(df, x=None, y=None, z=None, facet_row=None, facet_col=None, # Make faceted plot fig = make_subplots(rows=num_rows , cols=num_cols - , row_titles=facet_row_groups if facet_row else None - , column_titles=facet_col_groups if facet_col else None + , row_titles=row_titles + , column_titles=column_titles , x_title=x_title if x_title else None , y_title=y_title if y_title else None ) From 1679f28f5a09e3b156bef80002d5e0d6696c9038 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 19:24:50 -0400 Subject: [PATCH 36/57] Refactor generate_plot function to handle facet_row and facet_col groups properly --- lib/gear/plotting.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/gear/plotting.py b/lib/gear/plotting.py index ad0481f6..81e0121a 100644 --- a/lib/gear/plotting.py +++ b/lib/gear/plotting.py @@ -470,16 +470,12 @@ def generate_plot(df, x=None, y=None, z=None, facet_row=None, facet_col=None, # Map indexes for subplot ordering. Indexes start at 1 since plotting rows/cols start at 1 facet_row_groups = [] facet_col_groups = [] - row_titles = None - column_titles = None if facet_row: facet_row_groups = category_orders[facet_row] if facet_row in category_orders else df[facet_row].unique() - row_titles = facet_row_groups if facet_col: facet_col_groups = category_orders[facet_col] if facet_col in category_orders else df[facet_col].unique() - column_titles = facet_col_groups facet_row_indexes = {group: idx for idx, group in enumerate(facet_row_groups, start=1)} num_rows = len(facet_row_groups) if facet_row else 1 @@ -489,8 +485,8 @@ def generate_plot(df, x=None, y=None, z=None, facet_row=None, facet_col=None, # Make faceted plot fig = make_subplots(rows=num_rows , cols=num_cols - , row_titles=row_titles - , column_titles=column_titles + , row_titles=facet_row_groups + , column_titles=facet_col_groups , x_title=x_title if x_title else None , y_title=y_title if y_title else None ) From 8665c0cfd08128d07901b42166215c5d151eda61 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 19:26:49 -0400 Subject: [PATCH 37/57] Refactor generate_plot function to handle facet_row and facet_col groups properly --- lib/gear/plotting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gear/plotting.py b/lib/gear/plotting.py index 81e0121a..6d197ba1 100644 --- a/lib/gear/plotting.py +++ b/lib/gear/plotting.py @@ -485,8 +485,8 @@ def generate_plot(df, x=None, y=None, z=None, facet_row=None, facet_col=None, # Make faceted plot fig = make_subplots(rows=num_rows , cols=num_cols - , row_titles=facet_row_groups - , column_titles=facet_col_groups + , row_titles=list(facet_row_groups) + , column_titles=list(facet_col_groups) , x_title=x_title if x_title else None , y_title=y_title if y_title else None ) From d22035262e5b312652f98431a163de6695b79003 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 19:29:01 -0400 Subject: [PATCH 38/57] I suck at doing things --- lib/gear/plotting.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/gear/plotting.py b/lib/gear/plotting.py index 6d197ba1..31398a43 100644 --- a/lib/gear/plotting.py +++ b/lib/gear/plotting.py @@ -482,6 +482,10 @@ def generate_plot(df, x=None, y=None, z=None, facet_row=None, facet_col=None, facet_col_indexes = {group: idx for idx, group in enumerate(facet_col_groups, start=1)} num_cols = len(facet_col_groups) if facet_col else 1 + # Ensure facet_row_groups and facet_col_groups elements are cast as strings + facet_row_groups = [str(group) for group in facet_row_groups] + facet_col_groups = [str(group) for group in facet_col_groups] + # Make faceted plot fig = make_subplots(rows=num_rows , cols=num_cols From cfcfb331ee648f60a3363fbee80fd6b7db3c8907 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 19:30:28 -0400 Subject: [PATCH 39/57] Hopefully the last time to fix this edge case --- lib/gear/plotting.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/gear/plotting.py b/lib/gear/plotting.py index 31398a43..fc5ee254 100644 --- a/lib/gear/plotting.py +++ b/lib/gear/plotting.py @@ -472,20 +472,16 @@ def generate_plot(df, x=None, y=None, z=None, facet_row=None, facet_col=None, facet_col_groups = [] if facet_row: - facet_row_groups = category_orders[facet_row] if facet_row in category_orders else df[facet_row].unique() + facet_row_groups = category_orders[facet_row] if facet_row in category_orders else df[facet_row].unique().tolist() if facet_col: - facet_col_groups = category_orders[facet_col] if facet_col in category_orders else df[facet_col].unique() + facet_col_groups = category_orders[facet_col] if facet_col in category_orders else df[facet_col].unique().tolist() facet_row_indexes = {group: idx for idx, group in enumerate(facet_row_groups, start=1)} num_rows = len(facet_row_groups) if facet_row else 1 facet_col_indexes = {group: idx for idx, group in enumerate(facet_col_groups, start=1)} num_cols = len(facet_col_groups) if facet_col else 1 - # Ensure facet_row_groups and facet_col_groups elements are cast as strings - facet_row_groups = [str(group) for group in facet_row_groups] - facet_col_groups = [str(group) for group in facet_col_groups] - # Make faceted plot fig = make_subplots(rows=num_rows , cols=num_cols From 64407f9d6e15defeb3276ebe14623f454b3b1c84 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Mon, 12 Aug 2024 19:32:37 -0400 Subject: [PATCH 40/57] Fixing case where Int-casted group names are not being treated as strings --- lib/gear/plotting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gear/plotting.py b/lib/gear/plotting.py index fc5ee254..b321bd95 100644 --- a/lib/gear/plotting.py +++ b/lib/gear/plotting.py @@ -531,7 +531,7 @@ def generate_plot(df, x=None, y=None, z=None, facet_row=None, facet_col=None, # Each individual trace is a separate scalegroup to ensure plots are scaled correctly for violin plots new_plotting_args['scalegroup'] = name if isinstance(name, tuple): - new_plotting_args['scalegroup'] = "_".join(name) + new_plotting_args['scalegroup'] = "_".join(str(name)) # If color dataseries is present, add some special configurations if color_name: From ccf34097d88a2de81fe698590b552ce1a12ccbdf Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Tue, 13 Aug 2024 07:40:27 -0400 Subject: [PATCH 41/57] try/catch on lock files not found --- www/api/resources/projectr.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/www/api/resources/projectr.py b/www/api/resources/projectr.py index 0440923b..12a6b189 100644 --- a/www/api/resources/projectr.py +++ b/www/api/resources/projectr.py @@ -86,7 +86,11 @@ def remove_lock_file(fd, filepath): """Release the lock file.""" #fcntl.flock(fd, fcntl.LOCK_UN) fd.close() - Path(filepath).unlink() + try: + Path(filepath).unlink() + except FileNotFoundError: + # This is fine, as the lock file may have been removed by another process + pass def write_to_json(projections_dict, projection_json_file): with open(projection_json_file, 'w') as f: From 98ed357996d8e1c2e8292d83da87b57ac66b1704 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Tue, 13 Aug 2024 09:52:12 -0400 Subject: [PATCH 42/57] Error handling where genecart_id is not found --- www/api/resources/projectr.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/www/api/resources/projectr.py b/www/api/resources/projectr.py index 12a6b189..f02d13d4 100644 --- a/www/api/resources/projectr.py +++ b/www/api/resources/projectr.py @@ -248,7 +248,7 @@ def projectr_callback(dataset_id, genecart_id, projection_id, session_id, scope, , 'message': "Could not find gene cart in database" } - genecart.get_genes(); + genecart.get_genes() if not len(genecart.genes): return { @@ -601,12 +601,12 @@ def post(self, dataset_id): } # If legacy version exists, copy to current format. - if not genecart_id in projections_dict or "cart.{}".format(genecart_id) in projections_dict: - print("Copying legacy cart.{} to {} in the projection json file.".format(genecart_id, genecart_id), file=fh) + if (not genecart_id in projections_dict) and "cart.{}".format(genecart_id) in projections_dict: + print("Copying legacy cart.{} to {} in the projection json file.".format(genecart_id, genecart_id), file=sys.stderr) projections_dict[genecart_id] == projections_dict["cart.{}".format(genecart_id)] projections_dict.pop("cart.{}".format(genecart_id), None) # move legacy genecart projection stuff to new version - print("Moving legacy cart.{} contents to {} in the projection genecart directory.".format(genecart_id, genecart_id), file=fh) + print("Moving legacy cart.{} contents to {} in the projection genecart directory.".format(genecart_id, genecart_id), file=sys.stderr) old_genecart_projection_json_file = build_projection_json_path("cart.{}".format(genecart_id), "genecart") old_genecart_projection_json_file.parent.rename(dataset_projection_json_file.parent) @@ -655,16 +655,19 @@ def post(self, dataset_id): common_genes = None genecart_genes = None dataset_genes = None - for config in projections_dict[genecart_id]: - # Handle legacy algorithm - if "is_pca" in config: - config["algorithm"] = "pca" if config["is_pca"] == 1 else "nmf" - - if algorithm == config["algorithm"]: - common_genes = config.get('num_common_genes', None) - genecart_genes = config.get('num_genecart_genes', -1) - dataset_genes = config.get('num_dataset_genes', -1) - break + + # Get projection info from the json file if it already exists + if genecart_id in projections_dict: + for config in projections_dict[genecart_id]: + # Handle legacy algorithm + if "is_pca" in config: + config["algorithm"] = "pca" if config["is_pca"] == 1 else "nmf" + + if algorithm == config["algorithm"]: + common_genes = config.get('num_common_genes', None) + genecart_genes = config.get('num_genecart_genes', -1) + dataset_genes = config.get('num_dataset_genes', -1) + break message = "" if common_genes: From a7c1f06490f6db8415fe0236a71c59d0ae4eb5bd Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Tue, 13 Aug 2024 13:21:14 -0400 Subject: [PATCH 43/57] script to remove duplicated layout displays --- bin/remove_duplicate_layout_displays.py | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 bin/remove_duplicate_layout_displays.py diff --git a/bin/remove_duplicate_layout_displays.py b/bin/remove_duplicate_layout_displays.py new file mode 100644 index 00000000..a72ad01c --- /dev/null +++ b/bin/remove_duplicate_layout_displays.py @@ -0,0 +1,30 @@ +#!/opt/bin/python + +# This is to fix an issue where some layouts have duplicated display members, if the user +# saved layouts in the layout arranger when the duplcation bug was active (https://github.com/IGS/gEAR/issues/768) + +import sys + +from pathlib import Path +lib_path = Path(__file__).resolve().parents[1].joinpath('lib') + +sys.path.append(str(lib_path)) + +import geardb + +conn = geardb.Connection() +cursor = conn.get_cursor() + +# https://www.tutorialspoint.com/mysql/mysql-delete-duplicate-records.htm +qry = """ +DELETE FROM layout_displays ld1 +INNER JOIN layout_displays ld2 +WHERE ld1.layout_id = ld2.layout_id +AND ld1.display_id = ld2.display_id +AND ld1.start_col = ld2.start_col +AND ld1.grid_width = ld2.grid_width +AND ld1.start_row = ld2.start_row +AND ld1.grid_height = ld2.grid_height +AND ld1.id > ld2.id +""" +cursor.execute(qry) \ No newline at end of file From cd10e4a81fe93b1db53ea6e7a90e729808af053d Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Tue, 13 Aug 2024 13:24:43 -0400 Subject: [PATCH 44/57] updates --- bin/remove_duplicate_layout_displays.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/bin/remove_duplicate_layout_displays.py b/bin/remove_duplicate_layout_displays.py index a72ad01c..84782325 100644 --- a/bin/remove_duplicate_layout_displays.py +++ b/bin/remove_duplicate_layout_displays.py @@ -15,9 +15,15 @@ conn = geardb.Connection() cursor = conn.get_cursor() +# print row count +qry = "SELECT COUNT(*) FROM layout_displays" +cursor.execute(qry) +row_count = cursor.fetchone()[0] +print("Row count before deletion: {}".format(row_count)) + # https://www.tutorialspoint.com/mysql/mysql-delete-duplicate-records.htm qry = """ -DELETE FROM layout_displays ld1 +DELETE ld1 FROM layout_displays ld1 INNER JOIN layout_displays ld2 WHERE ld1.layout_id = ld2.layout_id AND ld1.display_id = ld2.display_id @@ -27,4 +33,15 @@ AND ld1.grid_height = ld2.grid_height AND ld1.id > ld2.id """ -cursor.execute(qry) \ No newline at end of file +cursor.execute(qry) + +cursor.commit() + +# print row count +qry = "SELECT COUNT(*) FROM layout_displays" +cursor.execute(qry) +row_count = cursor.fetchone()[0] +print("Row count after deletion: {}".format(row_count)) + +cursor.close() +conn.close() \ No newline at end of file From fe96daaec415268798336de364a7c4adb0e64f58 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Tue, 13 Aug 2024 13:26:38 -0400 Subject: [PATCH 45/57] syntax fix --- bin/remove_duplicate_layout_displays.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/remove_duplicate_layout_displays.py b/bin/remove_duplicate_layout_displays.py index 84782325..a500d40d 100644 --- a/bin/remove_duplicate_layout_displays.py +++ b/bin/remove_duplicate_layout_displays.py @@ -35,7 +35,7 @@ """ cursor.execute(qry) -cursor.commit() +conn.commit() # print row count qry = "SELECT COUNT(*) FROM layout_displays" From bc06d9390252b788ce14640968a46af59ced5c83 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Tue, 13 Aug 2024 13:53:33 -0400 Subject: [PATCH 46/57] Note to myself --- www/api/resources/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/www/api/resources/common.py b/www/api/resources/common.py index a83b8418..4acd34c5 100644 --- a/www/api/resources/common.py +++ b/www/api/resources/common.py @@ -46,6 +46,8 @@ def create_projection_adata(dataset_adata, dataset_id, projection_id): # Associate with a filename to ensure AnnData is read in "backed" mode # This creates the h5ad file if it does not exist + # TODO: If too many processes read from this file, it can throw a BlockingIOError. Eventually we should + # handle this by creating a copy of the file for each process, like a tempfile. projection_adata.filename = projection_adata_path return projection_adata From e267cd4a4de1f3026e5bbec4bb45f38b108824c0 Mon Sep 17 00:00:00 2001 From: adkinsrs Date: Tue, 13 Aug 2024 15:09:35 -0400 Subject: [PATCH 47/57] #739 - Added text to single and multigene curators --- www/dataset_curator.html | 1 + www/js/curator_common.js | 6 ++++++ www/js/dataset_curator.js | 4 ++++ www/js/multigene_curator.js | 7 ++++++- www/multigene_curator.html | 5 +++++ 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/www/dataset_curator.html b/www/dataset_curator.html index af05b4a4..abb47257 100644 --- a/www/dataset_curator.html +++ b/www/dataset_curator.html @@ -289,6 +289,7 @@

Dataset:

Analysis: Primary analysis

Number of selected observations:

+

Gene: