From b968044d52c247f9f45da2a4395eeaaf9c430bdb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 28 Jun 2024 08:08:25 -0400 Subject: [PATCH 01/12] Refactor method for reuse. --- lib/tool_shed/managers/trs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/tool_shed/managers/trs.py b/lib/tool_shed/managers/trs.py index 3c5cfb9e89e1..d77fdc7334ee 100644 --- a/lib/tool_shed/managers/trs.py +++ b/lib/tool_shed/managers/trs.py @@ -112,7 +112,7 @@ def trs_tool_id_to_repository_metadata( app = trans.app versions: Dict[str, RepositoryMetadata] = get_repository_metadata_by_tool_version(app, repository, tool_id) if not versions: - return None + raise ObjectNotFound() return repository, versions @@ -121,8 +121,6 @@ def get_tool(trans: ProvidesRepositoriesContext, trs_tool_id: str) -> Tool: guid = decode_identifier(trans.repositories_hostname, trs_tool_id) guid = remove_protocol_and_user_from_clone_url(guid) repo_metadata = trs_tool_id_to_repository_metadata(trans, trs_tool_id) - if not repo_metadata: - raise ObjectNotFound() repository, metadata_by_version = repo_metadata repo_owner = repository.user.username From 281b7cf2f14fbdbe35b51e14a8a087abf71ecb07 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 30 Nov 2022 11:13:32 -0500 Subject: [PATCH 02/12] Bug in color parameter parsing... --- lib/galaxy/tools/parameters/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 4a20224a7953..9b5a8f5fde9c 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -845,7 +845,7 @@ def __init__(self, tool, input_source): input_source = ensure_input_source(input_source) super().__init__(tool, input_source) self.value = input_source.get("value", "#000000") - self.rgb = input_source.get("rgb", False) + self.rgb = input_source.get_bool("rgb", False) def get_initial_value(self, trans, other_values): if self.value is not None: From 301e1fc3fce5824dd11012e731752012cb00601a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 7 Jul 2024 13:41:39 -0400 Subject: [PATCH 03/12] Remove duplication in test_parsing.py. --- test/unit/tool_util/test_parsing.py | 75 +++++++++++++++-------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/test/unit/tool_util/test_parsing.py b/test/unit/tool_util/test_parsing.py index 7550bea601ab..a1d5c9732d2a 100644 --- a/test/unit/tool_util/test_parsing.py +++ b/test/unit/tool_util/test_parsing.py @@ -250,9 +250,10 @@ def _tool_source(self): def _get_tool_source(self, source_file_name=None, source_contents=None, macro_contents=None): if source_file_name is None: - source_file_name = self.source_file_name + source_file_name = self._get_source_file_name() if source_contents is None: source_contents = self.source_contents + return get_test_tool_source( source_file_name, source_contents, @@ -260,6 +261,10 @@ def _get_tool_source(self, source_file_name=None, source_contents=None, macro_co self.temp_directory, ) + def _get_source_file_name(self) -> str: + assert self.source_file_name + return self.source_file_name + class TestXmlExpressionLoader(BaseLoaderTestCase): source_file_name = "expression.xml" @@ -672,9 +677,16 @@ def test_tool_type(self): assert tool_module[1] == "BuildListCollectionTool" -class TestExpressionTestToolLoader(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/expression_null_handling_boolean.xml") - source_contents = None +class FunctionalTestToolTestCase(BaseLoaderTestCase): + test_path: str + source_contents: None + + def _get_source_file_name(self) -> str: + return os.path.join(galaxy_directory(), "test/functional/tools", self.test_path) + + +class TestExpressionTestToolLoader(FunctionalTestToolTestCase): + test_path = "expression_null_handling_boolean.xml" def test_test(self): test_dicts = self._tool_source.parse_tests_to_dict()["tests"] @@ -701,9 +713,8 @@ def test_test(self): assert output0["attributes"]["object"] is None -class TestDefaultDataTestToolLoader(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/for_workflows/cat_default.xml") - source_contents = None +class TestDefaultDataTestToolLoader(FunctionalTestToolTestCase): + test_path = "for_workflows/cat_default.xml" def test_input_parsing(self): input_pages = self._tool_source.parse_input_pages() @@ -719,9 +730,8 @@ def test_input_parsing(self): assert default_dict["location"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed" -class TestDefaultCollectionDataTestToolLoader(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/collection_paired_default.xml") - source_contents = None +class TestDefaultCollectionDataTestToolLoader(FunctionalTestToolTestCase): + test_path = "collection_paired_default.xml" def test_input_parsing(self): input_pages = self._tool_source.parse_input_pages() @@ -745,9 +755,8 @@ def test_input_parsing(self): assert element1["location"] == "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.fasta" -class TestDefaultNestedCollectionDataTestToolLoader(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/collection_nested_default.xml") - source_contents = None +class TestDefaultNestedCollectionDataTestToolLoader(FunctionalTestToolTestCase): + test_path = "collection_nested_default.xml" def test_input_parsing(self): input_pages = self._tool_source.parse_input_pages() @@ -774,9 +783,8 @@ def test_input_parsing(self): assert elements01["identifier"] == "reverse" -class TestExpressionOutputDataToolLoader(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/expression_pick_larger_file.xml") - source_contents = None +class TestExpressionOutputDataToolLoader(FunctionalTestToolTestCase): + test_path = "expression_pick_larger_file.xml" def test_output_parsing(self): outputs, _ = self._tool_source.parse_outputs(None) @@ -808,9 +816,8 @@ def test_action(self): assert action[1] == "ExportHistoryToolAction" -class TestCollection(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/collection_two_paired.xml") - source_contents = None +class TestCollection(FunctionalTestToolTestCase): + test_path = "collection_two_paired.xml" def test_tests(self): tests_dict = self._tool_source.parse_tests_to_dict() @@ -822,27 +829,24 @@ def test_tests(self): assert len(output_collections) == 0 -class TestCollectionOutputXml(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/collection_creates_pair.xml") - source_contents = None +class TestCollectionOutputXml(FunctionalTestToolTestCase): + test_path = "collection_creates_pair.xml" def test_tests(self): outputs, output_collections = self._tool_source.parse_outputs(None) assert len(output_collections) == 1 -class TestCollectionOutputYaml(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/collection_creates_pair_y.yml") - source_contents = None +class TestCollectionOutputYaml(FunctionalTestToolTestCase): + test_path = "collection_creates_pair_y.yml" def test_tests(self): outputs, output_collections = self._tool_source.parse_outputs(None) assert len(output_collections) == 1 -class TestEnvironmentVariables(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/environment_variables.xml") - source_contents = None +class TestEnvironmentVariables(FunctionalTestToolTestCase): + test_path = "environment_variables.xml" def test_tests(self): tests_dict = self._tool_source.parse_tests_to_dict() @@ -850,9 +854,8 @@ def test_tests(self): assert len(tests) == 1 -class TestExpectations(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/detect_errors.xml") - source_contents = None +class TestExpectations(FunctionalTestToolTestCase): + test_path = "detect_errors.xml" def test_tests(self): tests_dict = self._tool_source.parse_tests_to_dict() @@ -863,9 +866,8 @@ def test_tests(self): assert len(test_0["stdout"]) == 2 -class TestExpectationsCommandVersion(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/job_properties.xml") - source_contents = None +class TestExpectationsCommandVersion(FunctionalTestToolTestCase): + test_path = "job_properties.xml" def test_tests(self): tests_dict = self._tool_source.parse_tests_to_dict() @@ -875,9 +877,8 @@ def test_tests(self): assert len(test_0["command_version"]) == 1 -class TestQcStdio(BaseLoaderTestCase): - source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/qc_stdout.xml") - source_contents = None +class TestQcStdio(FunctionalTestToolTestCase): + test_path = "qc_stdout.xml" def test_tests(self): exit, regexes = self._tool_source.parse_stdio() From 3eb03c0c1edec4748fc8e6ab0f6aca6aae759ed7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 4 Jul 2024 13:45:39 -0400 Subject: [PATCH 04/12] Pydantic models for tool outputs. --- .../tool_util/parser/output_collection_def.py | 77 +++++++++--- lib/galaxy/tool_util/parser/output_models.py | 115 ++++++++++++++++++ lib/galaxy/tool_util/parser/output_objects.py | 94 ++++++++++++-- lib/galaxy/tool_util/parser/xml.py | 4 +- lib/galaxy/util/__init__.py | 4 +- test/unit/tool_util/test_parsing.py | 87 ++++++++++++- 6 files changed, 353 insertions(+), 28 deletions(-) create mode 100644 lib/galaxy/tool_util/parser/output_models.py diff --git a/lib/galaxy/tool_util/parser/output_collection_def.py b/lib/galaxy/tool_util/parser/output_collection_def.py index baf61c520cdb..81675de7159f 100644 --- a/lib/galaxy/tool_util/parser/output_collection_def.py +++ b/lib/galaxy/tool_util/parser/output_collection_def.py @@ -2,9 +2,21 @@ dataset collection after jobs are finished. """ -from typing import List +import abc +from typing import ( + List, + Optional, +) from galaxy.util import asbool +from .output_models import ( + DatasetCollectionDescriptionT, + DiscoverViaT, + FilePatternDatasetCollectionDescription as FilePatternDatasetCollectionDescriptionModel, + SortCompT, + SortKeyT, + ToolProvidedMetadataDatasetCollection as ToolProvidedMetadataDatasetCollectionModel, +) from .util import is_dict DEFAULT_EXTRA_FILENAME_PATTERN = ( @@ -81,7 +93,15 @@ def dataset_collection_description(**kwargs): return FilePatternDatasetCollectionDescription(**kwargs) -class DatasetCollectionDescription: +class DatasetCollectionDescription(metaclass=abc.ABCMeta): + discover_via: DiscoverViaT + default_ext: Optional[str] + default_visible: bool + assign_primary_output: bool + directory: Optional[str] + recurse: bool + match_relative_path: bool + def __init__(self, **kwargs): self.default_dbkey = kwargs.get("dbkey", INPUT_DBKEY_TOKEN) self.default_ext = kwargs.get("ext", None) @@ -91,9 +111,9 @@ def __init__(self, **kwargs): self.assign_primary_output = asbool(kwargs.get("assign_primary_output", False)) self.directory = kwargs.get("directory", None) self.recurse = False - self.match_relative_path = kwargs.get("match_relative_path", False) + self.match_relative_path = asbool(kwargs.get("match_relative_path", False)) - def to_dict(self): + def _common_model_props(self): return { "discover_via": self.discover_via, "dbkey": self.default_dbkey, @@ -105,6 +125,12 @@ def to_dict(self): "match_relative_path": self.match_relative_path, } + @abc.abstractmethod + def to_model(self) -> DatasetCollectionDescriptionT: ... + + def to_dict(self) -> dict: + return self.to_model().model_dump() + @property def discover_patterns(self) -> List[str]: return [] @@ -113,9 +139,27 @@ def discover_patterns(self) -> List[str]: class ToolProvidedMetadataDatasetCollection(DatasetCollectionDescription): discover_via = "tool_provided_metadata" + def to_model(self) -> ToolProvidedMetadataDatasetCollectionModel: + return ToolProvidedMetadataDatasetCollectionModel( + discover_via=self.discover_via, + dbkey=self.default_dbkey, + format=self.default_ext, + visible=self.default_visible, + assign_primary_output=self.assign_primary_output, + directory=self.directory, + recurse=self.recurse, + match_relative_path=self.match_relative_path, + ) + + def to_dict(self) -> dict: + return self.to_model().model_dump() + class FilePatternDatasetCollectionDescription(DatasetCollectionDescription): discover_via = "pattern" + sort_key: SortKeyT + sort_comp: SortCompT + pattern: str def __init__(self, **kwargs): super().__init__(**kwargs) @@ -140,18 +184,21 @@ def __init__(self, **kwargs): self.sort_key = sort_by self.sort_comp = sort_comp - def to_dict(self): - as_dict = super().to_dict() - as_dict.update( - { - "sort_key": self.sort_key, - "sort_comp": self.sort_comp, - "pattern": self.pattern, - "recurse": self.recurse, - "sort_by": self.sort_by, - } + def to_model(self) -> FilePatternDatasetCollectionDescriptionModel: + return FilePatternDatasetCollectionDescriptionModel( + discover_via=self.discover_via, + dbkey=self.default_dbkey, + format=self.default_ext, + visible=self.default_visible, + assign_primary_output=self.assign_primary_output, + directory=self.directory, + recurse=self.recurse, + match_relative_path=self.match_relative_path, + sort_key=self.sort_key, + sort_comp=self.sort_comp, + pattern=self.pattern, + sort_by=self.sort_by, ) - return as_dict @property def discover_patterns(self) -> List[str]: diff --git a/lib/galaxy/tool_util/parser/output_models.py b/lib/galaxy/tool_util/parser/output_models.py new file mode 100644 index 000000000000..3c190dbecadf --- /dev/null +++ b/lib/galaxy/tool_util/parser/output_models.py @@ -0,0 +1,115 @@ +"""Modern pydantic based descriptions of Galaxy tool output objects. + +output_objects.py is still used for internals and contain references to the actual tool object +but the goal here is to switch to using these overtime at least for external APIs and in library +code where actual tool objects aren't created. +""" + +from typing import ( + List, + Optional, + Union, +) + +from pydantic import ( + BaseModel, + Field, +) +from typing_extensions import ( + Annotated, + Literal, +) + +from .interface import ToolSource + + +class ToolOutputBaseModel(BaseModel): + name: str + label: Optional[str] + hidden: bool + + +class ToolOutputDataset(ToolOutputBaseModel): + type: Literal["data"] + format: str + format_source: Optional[str] + metadata_source: Optional[str] + discover_datasets: Optional[List["DatasetCollectionDescriptionT"]] + + +class ToolOutputCollectionStructure(BaseModel): + collection_type: Optional[str] + collection_type_source: Optional[str] + collection_type_from_rules: Optional[str] + structured_like: Optional[str] + discover_datasets: Optional[List["DatasetCollectionDescriptionT"]] + + +class ToolOutputCollection(ToolOutputBaseModel): + type: Literal["collection"] + structure: ToolOutputCollectionStructure + + +class ToolOutputSimple(ToolOutputBaseModel): + pass + + +class ToolOutputText(ToolOutputSimple): + type: Literal["text"] + + +class ToolOutputInteger(ToolOutputSimple): + type: Literal["integer"] + + +class ToolOutputFloat(ToolOutputSimple): + type: Literal["float"] + + +class ToolOutputBoolean(ToolOutputSimple): + type: Literal["boolean"] + + +DiscoverViaT = Literal["tool_provided_metadata", "pattern"] +SortKeyT = Literal["filename", "name", "designation", "dbkey"] +SortCompT = Literal["lexical", "numeric"] + + +class DatasetCollectionDescription(BaseModel): + discover_via: DiscoverViaT + format: Optional[str] + visible: bool + assign_primary_output: bool + directory: Optional[str] + recurse: bool + match_relative_path: bool + + +class ToolProvidedMetadataDatasetCollection(DatasetCollectionDescription): + discover_via: Literal["tool_provided_metadata"] + + +class FilePatternDatasetCollectionDescription(DatasetCollectionDescription): + discover_via: Literal["pattern"] + sort_key: SortKeyT + sort_comp: SortCompT + pattern: str + + +DatasetCollectionDescriptionT = Union[FilePatternDatasetCollectionDescription, ToolProvidedMetadataDatasetCollection] + + +ToolOutputT = Union[ + ToolOutputDataset, ToolOutputCollection, ToolOutputText, ToolOutputInteger, ToolOutputFloat, ToolOutputBoolean +] +ToolOutput = Annotated[ToolOutputT, Field(discriminator="type")] + + +def from_tool_source(tool_source: ToolSource) -> List[ToolOutput]: + tool_outputs, tool_output_collections = tool_source.parse_outputs(object()) + outputs = [] + for tool_output in tool_outputs.values(): + outputs.append(tool_output.to_model()) + # for tool_output_collection in tool_output_collections.values(): + # outputs.append(tool_output_collection.to_model()) + return outputs diff --git a/lib/galaxy/tool_util/parser/output_objects.py b/lib/galaxy/tool_util/parser/output_objects.py index 27d06ca29c2b..63148c1fb946 100644 --- a/lib/galaxy/tool_util/parser/output_objects.py +++ b/lib/galaxy/tool_util/parser/output_objects.py @@ -3,6 +3,8 @@ Dict, List, Optional, + Type, + Union, ) from typing_extensions import TypedDict @@ -14,6 +16,16 @@ dataset_collector_descriptions_from_output_dict, DatasetCollectionDescription, ) +from .output_models import ( + ToolOutputBoolean as ToolOutputBooleanModel, + ToolOutputCollection as ToolOutputCollectionModel, + ToolOutputCollectionStructure as ToolOutputCollectionStructureModel, + ToolOutputDataset as ToolOutputDataModel, + ToolOutputFloat as ToolOutputFloatModel, + ToolOutputInteger as ToolOutputIntegerModel, + ToolOutputT as ToolOutputModel, + ToolOutputText as ToolOutputTextModel, +) class ChangeFormatModel(TypedDict): @@ -25,6 +37,10 @@ class ChangeFormatModel(TypedDict): class ToolOutputBase(Dictifiable): + name: str + label: Optional[str] + hidden: bool + def __init__( self, name: str, @@ -128,9 +144,24 @@ def to_dict(self, view="collection", value_mapper=None, app=None): as_dict["edam_format"] = edam_format edam_data = app.datatypes_registry.edam_data.get(self.format) as_dict["edam_data"] = edam_data - as_dict["discover_datasets"] = [d.to_dict() for d in self.dataset_collector_descriptions] + if self.dataset_collector_descriptions: + as_dict["discover_datasets"] = [d.to_dict() for d in self.dataset_collector_descriptions] + else: + as_dict["discover_datasets"] = [] return as_dict + def to_model(self) -> ToolOutputDataModel: + return ToolOutputDataModel( + type="data", + name=self.name, + label=self.label, + hidden=self.hidden, + format=self.format, + format_source=self.format_source, + metadata_source=self.metadata_source or None, # model is decorated as Optional + discover_datasets=[d.to_model() for d in self.dataset_collector_descriptions], + ) + @staticmethod def from_dict(name: str, output_dict: Dict[str, Any], tool: Optional[object] = None) -> "ToolOutput": output = ToolOutput(name) @@ -175,6 +206,31 @@ def __init__(self, name, output_type, from_expression, label=None, filters=None, self.implicit = False self.from_work_dir = None + def to_model(self) -> ToolOutputModel: + model_class: Union[ + Type[ToolOutputIntegerModel], + Type[ToolOutputFloatModel], + Type[ToolOutputBooleanModel], + Type[ToolOutputTextModel], + ] + model_type = self.output_type + if self.output_type == "integer": + model_class = ToolOutputIntegerModel + elif self.output_type == "float": + model_class = ToolOutputFloatModel + elif self.output_type == "bool": + model_class = ToolOutputBooleanModel + model_type = "boolean" + elif self.output_type == "text": + model_class = ToolOutputTextModel + assert model_class + return model_class( + type=model_type, + name=self.name, + label=self.label, + hidden=self.hidden, + ) + class ToolOutputCollection(ToolOutputBase): """ @@ -300,6 +356,15 @@ def to_dict(self, view="collection", value_mapper=None, app=None): as_dict["structure"] = self.structure.to_dict() return as_dict + def to_model(self) -> ToolOutputCollectionModel: + return ToolOutputCollectionModel( + type="collection", + name=self.name, + label=self.label, + hidden=self.hidden, + structure=self.structure.to_model(), + ) + @staticmethod def from_dict(name, output_dict, tool=None) -> "ToolOutputCollection": structure = ToolOutputCollectionStructure.from_dict(output_dict["structure"]) @@ -323,6 +388,13 @@ def output_discover_patterns(self) -> List[str]: class ToolOutputCollectionStructure: + collection_type: Optional[str] + collection_type_source: Optional[str] + collection_type_from_rules: Optional[str] + structured_like: Optional[str] + dataset_collector_descriptions: Optional[List[DatasetCollectionDescription]] + dynamic: bool + def __init__( self, collection_type: Optional[str], @@ -366,13 +438,19 @@ def collection_prototype(self, inputs, type_registry): return collection_prototype def to_dict(self): - return { - "collection_type": self.collection_type, - "collection_type_source": self.collection_type_source, - "collection_type_from_rules": self.collection_type_from_rules, - "structured_like": self.structured_like, - "discover_datasets": [d.to_dict() for d in self.dataset_collector_descriptions], - } + return self.to_model().dict() + + def to_model(self) -> ToolOutputCollectionStructureModel: + discover_datasets = [] + if self.dataset_collector_descriptions: + discover_datasets = [d.to_model() for d in self.dataset_collector_descriptions] + return ToolOutputCollectionStructureModel( + collection_type=self.collection_type, + collection_type_source=self.collection_type_source, + collection_type_from_rules=self.collection_type_from_rules, + structured_like=self.structured_like, + discover_datasets=discover_datasets, + ) @staticmethod def from_dict(as_dict) -> "ToolOutputCollectionStructure": diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 57d22e34750e..bf4b04716c27 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -535,7 +535,7 @@ def _parse_output( output.default_identifier_source = data_elem.get("default_identifier_source", "None") output.metadata_source = data_elem.get("metadata_source", default_metadata_source) output.parent = data_elem.get("parent", None) - output.label = xml_text(data_elem, "label") + output.label = xml_text(data_elem, "label", None) output.count = int(data_elem.get("count", 1)) output.filters = data_elem.findall("filter") output.tool = tool @@ -561,7 +561,7 @@ def _parse_expression_output(self, output_elem, tool, **kwds): from_expression, ) output.path = output_elem.get("value") - output.label = xml_text(output_elem, "label") + output.label = xml_text(output_elem, "label", None) output.hidden = string_as_bool(output_elem.get("hidden", "")) output.actions = ToolOutputActionGroup(output, output_elem.find("actions")) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index ca844c2358fc..a0ba3205225f 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -996,7 +996,7 @@ def update(self, values): self.__dict__.update(values) -def xml_text(root, name=None): +def xml_text(root, name=None, default=""): """Returns the text inside an element""" if name is not None: # Try attribute first @@ -1011,7 +1011,7 @@ def xml_text(root, name=None): text = "".join(elem.text.splitlines()) return text.strip() # No luck, return empty string - return "" + return default def parse_resource_parameters(resource_param_file): diff --git a/test/unit/tool_util/test_parsing.py b/test/unit/tool_util/test_parsing.py index a1d5c9732d2a..4832c7b73d9b 100644 --- a/test/unit/tool_util/test_parsing.py +++ b/test/unit/tool_util/test_parsing.py @@ -3,9 +3,21 @@ import shutil import tempfile from math import isinf -from typing import Optional +from typing import ( + cast, + List, + Optional, + Type, + TypeVar, +) from galaxy.tool_util.parser.factory import get_tool_source +from galaxy.tool_util.parser.output_models import ( + from_tool_source, + ToolOutput, + ToolOutputCollection, + ToolOutputDataset, +) from galaxy.util import galaxy_directory from galaxy.util.unittest import TestCase @@ -248,6 +260,10 @@ def tearDown(self): def _tool_source(self): return self._get_tool_source() + @property + def _output_models(self) -> List[ToolOutput]: + return from_tool_source(self._tool_source) + def _get_tool_source(self, source_file_name=None, source_contents=None, macro_contents=None): if source_file_name is None: source_file_name = self._get_source_file_name() @@ -388,6 +404,16 @@ def test_tests(self): assert attributes1["compare"] == "sim_size" assert attributes1["lines_diff"] == 4 + def test_output_models(self): + output_models = self._output_models + assert len(output_models) == 1 + output_model = output_models[0] + assert output_model.name == "out1" + assert not output_model.hidden + assert output_model.label is None + output_dataset_model = assert_output_model_of_type(output_model, ToolOutputDataset) + assert output_dataset_model.metadata_source is None + def test_xrefs(self): xrefs = self._tool_source.parse_xrefs() assert xrefs == [{"value": "bwa", "reftype": "bio.tools"}] @@ -666,6 +692,17 @@ def test_outputs(self): assert len(outputs) == 1 assert len(output_collections) == 1 + def test_output_models(self): + output_models = self._output_models + assert len(output_models) == 1 + output_model = output_models[0] + assert output_model.name == "output" + assert not output_model.hidden + assert output_model.label == "${input.name} (re-organized)" + output_collection_model = assert_output_model_of_type(output_model, ToolOutputCollection) + structure = output_collection_model.structure + assert structure.collection_type_from_rules == "rules" + class TestBuildListToolLoader(BaseLoaderTestCase): source_file_name = os.path.join(galaxy_directory(), "lib/galaxy/tools/build_list.xml") @@ -712,6 +749,14 @@ def test_test(self): assert "object" in output0["attributes"] assert output0["attributes"]["object"] is None + def test_output_models(self): + output_models = self._output_models + assert len(output_models) == 1 + output_model = output_models[0] + assert output_model.name == "bool_out" + assert not output_model.hidden + assert output_model.label is None + class TestDefaultDataTestToolLoader(FunctionalTestToolTestCase): test_path = "for_workflows/cat_default.xml" @@ -886,3 +931,43 @@ def test_tests(self): assert len(regexes) == 2 regex = regexes[0] assert regex.error_level == 1.1 + + +class TestCollectionCatGroupTag(FunctionalTestToolTestCase): + test_path = "collection_cat_group_tag.xml" + + def test_output_models(self): + output_models = self._output_models + assert len(output_models) == 1 + output_model = output_models[0] + assert output_model.name == "out_file1" + assert not output_model.hidden + assert output_model.label is None + output_dataset_model = assert_output_model_of_type(output_model, ToolOutputDataset) + assert output_dataset_model.metadata_source == "input1" + + +class TestToolProvidedMetadata2(FunctionalTestToolTestCase): + test_path = "tool_provided_metadata_2.xml" + + def test_output_models(self): + output_models = self._output_models + assert len(output_models) == 1 + output_model = output_models[0] + assert output_model.name == "sample" + assert not output_model.hidden + assert output_model.label is None + output_dataset_model = assert_output_model_of_type(output_model, ToolOutputDataset) + assert output_dataset_model.metadata_source is None + discover_datasets = output_dataset_model.discover_datasets or [] + assert len(discover_datasets) == 1 + discover_datasets_0 = discover_datasets[0] + assert discover_datasets_0.discover_via == "pattern" + + +T = TypeVar("T") + + +def assert_output_model_of_type(obj, clazz: Type[T]) -> T: + assert isinstance(obj, clazz) + return cast(T, obj) From 32a968f01cccba51da84b2f278144308d242e57b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 4 Jul 2024 14:01:25 -0400 Subject: [PATCH 05/12] More tool parsing abstractions, for greater reuse outside main app. --- lib/galaxy/managers/citations.py | 48 ++++++++++++------- .../tool_util/ontologies/ontology_data.py | 7 ++- lib/galaxy/tool_util/parser/__init__.py | 2 + lib/galaxy/tool_util/parser/interface.py | 32 ++++++++++--- lib/galaxy/tool_util/parser/xml.py | 46 +++++++++++++++--- lib/galaxy/tool_util/parser/yaml.py | 26 +++++----- lib/galaxy/tools/__init__.py | 34 ++++++------- test/unit/app/tools/test_citations.py | 6 ++- 8 files changed, 135 insertions(+), 66 deletions(-) diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py index a76e38aea9b7..e23403e8a3c4 100644 --- a/lib/galaxy/managers/citations.py +++ b/lib/galaxy/managers/citations.py @@ -1,10 +1,17 @@ import functools import logging +from typing import ( + Dict, + Optional, + Type, + Union, +) from beaker.cache import CacheManager from beaker.util import parse_cache_config_options from galaxy.structured_app import BasicSharedApp +from galaxy.tool_util.parser.interface import Citation from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, requests, @@ -12,6 +19,9 @@ log = logging.getLogger(__name__) +CitationT = Union["BibtexCitation", "DoiCitation"] +OptionalCitationT = Optional[CitationT] + class CitationsManager: def __init__(self, app: BasicSharedApp) -> None: @@ -29,8 +39,8 @@ def citations_for_tool_ids(self, tool_ids): citation_collection.add(citation) return citation_collection.citations - def parse_citation(self, citation_elem): - return parse_citation(citation_elem, self) + def parse_citation(self, citation_model: Citation) -> OptionalCitationT: + return parse_citation(citation_model, self) def _get_tool(self, tool_id): tool = self.app.toolbox.get_tool(tool_id) @@ -67,19 +77,19 @@ def get_bibtex(self, doi): return self._cache.get(key=doi, createfunc=createfunc) -def parse_citation(elem, citation_manager): +def parse_citation(citation_model: Citation, citation_manager) -> OptionalCitationT: """ Parse an abstract citation entry from the specified XML element. """ - citation_type = elem.attrib.get("type", None) + citation_type = citation_model.type citation_class = CITATION_CLASSES.get(citation_type, None) if not citation_class: log.warning(f"Unknown or unspecified citation type: {citation_type}") return None try: - citation = citation_class(elem, citation_manager) + citation = citation_class(citation_model, citation_manager) except Exception as e: - raise Exception(f"Invalid citation of type '{citation_type}' with content '{elem.text}': {e}") + raise Exception(f"Invalid citation of type '{citation_type}' with content '{citation_model.content}': {e}") return citation @@ -126,20 +136,22 @@ def has_doi(self): class BibtexCitation(BaseCitation): - def __init__(self, elem, citation_manager): - self.raw_bibtex = elem.text.strip() + def __init__(self, citation_model: Citation, citation_manager: CitationsManager): + self.raw_bibtex = citation_model.content - def to_bibtex(self): + def to_bibtex(self) -> str: return self.raw_bibtex +BIBTEX_UNSET = object() + + class DoiCitation(BaseCitation): - BIBTEX_UNSET = object() - def __init__(self, elem, citation_manager): - self.__doi = elem.text.strip() + def __init__(self, citation_model: Citation, citation_manager: CitationsManager): + self.__doi = citation_model.content self.doi_cache = citation_manager.doi_cache - self.raw_bibtex = DoiCitation.BIBTEX_UNSET + self.raw_bibtex = BIBTEX_UNSET def has_doi(self): return True @@ -147,23 +159,23 @@ def has_doi(self): def doi(self): return self.__doi - def to_bibtex(self): - if self.raw_bibtex is DoiCitation.BIBTEX_UNSET: + def to_bibtex(self) -> str: + if self.raw_bibtex is BIBTEX_UNSET: try: self.raw_bibtex = self.doi_cache.get_bibtex(self.__doi) except Exception: log.debug("Failed to fetch bibtex for DOI %s", self.__doi) - if self.raw_bibtex is DoiCitation.BIBTEX_UNSET: + if self.raw_bibtex is BIBTEX_UNSET: return f"""@MISC{{{self.__doi}, DOI = {{{self.__doi}}}, note = {{Failed to fetch BibTeX for DOI.}} }}""" else: - return self.raw_bibtex + return str(self.raw_bibtex) -CITATION_CLASSES = dict( +CITATION_CLASSES: Dict[str, Type[CitationT]] = dict( bibtex=BibtexCitation, doi=DoiCitation, ) diff --git a/lib/galaxy/tool_util/ontologies/ontology_data.py b/lib/galaxy/tool_util/ontologies/ontology_data.py index 9903031c1790..67822130e428 100644 --- a/lib/galaxy/tool_util/ontologies/ontology_data.py +++ b/lib/galaxy/tool_util/ontologies/ontology_data.py @@ -8,7 +8,10 @@ ) from galaxy.tool_util.biotools import BiotoolsMetadataSource -from galaxy.tool_util.parser import ToolSource +from galaxy.tool_util.parser import ( + ToolSource, + XrefDict, +) from galaxy.util.resources import resource_string @@ -46,7 +49,7 @@ def _read_ontology_data_text(filename: str) -> str: class OntologyData(NamedTuple): - xrefs: List[Dict[str, str]] + xrefs: List[XrefDict] edam_operations: Optional[List[str]] edam_topics: Optional[List[str]] diff --git a/lib/galaxy/tool_util/parser/__init__.py b/lib/galaxy/tool_util/parser/__init__.py index 509a6c066b31..0a67935df53c 100644 --- a/lib/galaxy/tool_util/parser/__init__.py +++ b/lib/galaxy/tool_util/parser/__init__.py @@ -9,6 +9,7 @@ from .interface import ( RequiredFiles, ToolSource, + XrefDict, ) from .output_objects import ToolOutputCollectionPart @@ -19,4 +20,5 @@ "RequiredFiles", "ToolOutputCollectionPart", "ToolSource", + "XrefDict", ) diff --git a/lib/galaxy/tool_util/parser/interface.py b/lib/galaxy/tool_util/parser/interface.py index 8cb859deee1e..f9d7f6440968 100644 --- a/lib/galaxy/tool_util/parser/interface.py +++ b/lib/galaxy/tool_util/parser/interface.py @@ -17,6 +17,7 @@ ) import packaging.version +from pydantic import BaseModel from typing_extensions import TypedDict from galaxy.util.path import safe_walk @@ -61,6 +62,16 @@ class ToolSourceTests(TypedDict): tests: List[ToolSourceTest] +class XrefDict(TypedDict): + value: str + reftype: str + + +class Citation(BaseModel): + type: str + content: str + + class ToolSource(metaclass=ABCMeta): """This interface represents an abstract source to parse tool information from. @@ -76,7 +87,7 @@ def parse_id(self) -> Optional[str]: """ @abstractmethod - def parse_version(self): + def parse_version(self) -> Optional[str]: """Parse a version describing the abstract tool.""" def parse_tool_module(self): @@ -98,12 +109,15 @@ def parse_tool_type(self): return None @abstractmethod - def parse_name(self): + def parse_name(self) -> str: """Parse a short name for tool (required).""" @abstractmethod - def parse_description(self): - """Parse a description for tool. Longer than name, shorted than help.""" + def parse_description(self) -> str: + """Parse a description for tool. Longer than name, shorted than help. + + We parse this out as "" if it isn't explicitly declared. + """ def parse_edam_operations(self) -> List[str]: """Parse list of edam operation codes.""" @@ -114,7 +128,7 @@ def parse_edam_topics(self) -> List[str]: return [] @abstractmethod - def parse_xrefs(self) -> List[Dict[str, str]]: + def parse_xrefs(self) -> List[XrefDict]: """Parse list of external resource URIs and types.""" def parse_display_interface(self, default): @@ -276,13 +290,17 @@ def parse_help(self) -> Optional[str]: """ @abstractmethod - def parse_profile(self): + def parse_profile(self) -> str: """Return tool profile version as Galaxy major e.g. 16.01 or 16.04.""" @abstractmethod - def parse_license(self): + def parse_license(self) -> Optional[str]: """Return license corresponding to tool wrapper.""" + def parse_citations(self) -> List[Citation]: + """Return a list of citations.""" + return [] + @abstractmethod def parse_python_template_version(self) -> Optional[packaging.version.Version]: """ diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index bf4b04716c27..6cc850307934 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -32,6 +32,7 @@ ) from .interface import ( AssertionList, + Citation, InputSource, PageSource, PagesSource, @@ -41,6 +42,7 @@ ToolSource, ToolSourceTest, ToolSourceTests, + XrefDict, ) from .output_actions import ToolOutputActionGroup from .output_collection_def import dataset_collector_descriptions_from_elem @@ -142,7 +144,7 @@ def mem_optimize(self): self.root = None self._xml_tree = None - def parse_version(self): + def parse_version(self) -> Optional[str]: return self.root.get("version", None) def parse_id(self): @@ -174,7 +176,7 @@ def parse_tool_type(self): return root.get("tool_type") def parse_name(self): - return self.root.get("name") + return self.root.get("name") or self.parse_id() def parse_edam_operations(self): edam_ops = self.root.find("edam_operations") @@ -188,17 +190,17 @@ def parse_edam_topics(self): return [] return [edam_topic.text for edam_topic in edam_topics.findall("edam_topic")] - def parse_xrefs(self): + def parse_xrefs(self) -> List[XrefDict]: xrefs = self.root.find("xrefs") if xrefs is None: return [] return [ - dict(value=xref.text.strip(), reftype=xref.attrib["type"]) + XrefDict(value=xref.text.strip(), reftype=str(xref.attrib["type"])) for xref in xrefs.findall("xref") - if xref.get("type") + if xref.get("type") and xref.text ] - def parse_description(self): + def parse_description(self) -> str: return xml_text(self.root, "description") def parse_display_interface(self, default): @@ -659,9 +661,24 @@ def parse_profile(self) -> str: # - Enable buggy interpreter attribute. return self.root.get("profile", "16.01") - def parse_license(self): + def parse_license(self) -> Optional[str]: return self.root.get("license") + def parse_citations(self) -> List[Citation]: + """Return a list of citations.""" + citations: List[Citation] = [] + root = self.root + citations_elem = root.find("citations") + if citations_elem is None: + return citations + + for citation_elem in citations_elem: + citation = parse_citation_elem(citation_elem) + if citation: + citations.append(citation) + + return citations + def parse_python_template_version(self): python_template_version = self.root.get("python_template_version") if python_template_version is not None: @@ -1372,3 +1389,18 @@ def __init__(self, tag): # legacy basic mode - provide compatible defaults self.attributes["split_size"] = 20 self.attributes["split_mode"] = "number_of_parts" + + +def parse_citation_elem(citation_elem: Element) -> Optional[Citation]: + if citation_elem.tag != "citation": + return None + + citation_type = citation_elem.attrib.get("type", None) + citation_raw_text = citation_elem.text + assert citation_raw_text + content = citation_raw_text.strip() + + return Citation( + type=citation_type, + content=content, + ) diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index 020f0019e99e..d6f135c027eb 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -22,6 +22,7 @@ ToolSource, ToolSourceTest, ToolSourceTests, + XrefDict, ) from .output_collection_def import dataset_collector_descriptions_from_output_dict from .output_objects import ( @@ -51,24 +52,27 @@ def parse_tool_type(self): def parse_id(self): return self.root_dict.get("id") - def parse_version(self): - return str(self.root_dict.get("version")) + def parse_version(self) -> Optional[str]: + version_raw = self.root_dict.get("version") + return str(version_raw) if version_raw is not None else None - def parse_name(self): - return self.root_dict.get("name") + def parse_name(self) -> str: + rval = self.root_dict.get("name") or self.parse_id() + assert rval + return str(rval) - def parse_description(self): + def parse_description(self) -> str: return self.root_dict.get("description", "") - def parse_edam_operations(self): + def parse_edam_operations(self) -> List[str]: return self.root_dict.get("edam_operations", []) - def parse_edam_topics(self): + def parse_edam_topics(self) -> List[str]: return self.root_dict.get("edam_topics", []) - def parse_xrefs(self): + def parse_xrefs(self) -> List[XrefDict]: xrefs = self.root_dict.get("xrefs", []) - return [dict(value=xref["value"], reftype=xref["type"]) for xref in xrefs if xref["type"]] + return [XrefDict(value=xref["value"], reftype=xref["type"]) for xref in xrefs if xref["type"]] def parse_sanitize(self): return self.root_dict.get("sanitize", True) @@ -191,10 +195,10 @@ def parse_tests_to_dict(self) -> ToolSourceTests: return rval - def parse_profile(self): + def parse_profile(self) -> str: return self.root_dict.get("profile", "16.04") - def parse_license(self): + def parse_license(self) -> Optional[str]: return self.root_dict.get("license") def parse_interactivetool(self): diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 680f26db1824..554275aaf6c4 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -566,6 +566,8 @@ def _create_tool_from_source(self, tool_source, **kwds): def create_dynamic_tool(self, dynamic_tool, **kwds): tool_format = dynamic_tool.tool_format tool_representation = dynamic_tool.value + if "name" not in tool_representation: + tool_representation["name"] = f"dynamic tool {dynamic_tool.uuid}" tool_source = get_tool_source_from_representation( tool_format=tool_format, tool_representation=tool_representation, @@ -1030,19 +1032,21 @@ def parse(self, tool_source: ToolSource, guid=None, dynamic=False): # Get the (user visible) name of the tool self.name = tool_source.parse_name() - if not self.name and dynamic: + if not self.name and dynamic and self.id: self.name = self.id if not dynamic and not self.name: raise Exception(f"Missing tool 'name' for tool with id '{self.id}' at '{tool_source}'") - self.version = tool_source.parse_version() - if not self.version: + version = tool_source.parse_version() + if not version: if profile < Version("16.04"): # For backward compatibility, some tools may not have versions yet. - self.version = "1.0.0" + version = "1.0.0" else: raise Exception(f"Missing tool 'version' for tool with id '{self.id}' at '{tool_source}'") + self.version = version + # Legacy feature, ignored by UI. self.force_history_refresh = False @@ -1476,22 +1480,12 @@ def parse_stdio(self, tool_source: ToolSource): self.stdio_regexes = regexes def _parse_citations(self, tool_source): - # TODO: Move following logic into ToolSource abstraction. - if not hasattr(tool_source, "root"): - return [] - - root = tool_source.root - citations: List[str] = [] - citations_elem = root.find("citations") - if citations_elem is None: - return citations - - for citation_elem in citations_elem: - if citation_elem.tag != "citation": - pass - citations_manager = getattr(self.app, "citations_manager", None) - if citations_manager is not None: - citation = citations_manager.parse_citation(citation_elem) + citation_models = tool_source.parse_citations() + citations_manager = getattr(self.app, "citations_manager", None) + citations = [] + if citations_manager is not None: + for citation_model in citation_models: + citation = citations_manager.parse_citation(citation_model) if citation: citations.append(citation) return citations diff --git a/test/unit/app/tools/test_citations.py b/test/unit/app/tools/test_citations.py index ab6b79292eec..9f6d7943acac 100644 --- a/test/unit/app/tools/test_citations.py +++ b/test/unit/app/tools/test_citations.py @@ -3,6 +3,7 @@ CitationCollection, parse_citation, ) +from galaxy.tool_util.parser.xml import parse_citation_elem from galaxy.util import parse_xml_string EXAMPLE_BIBTEX_CITATION = """@article{goecks2010galaxy, @@ -19,7 +20,10 @@ def test_parse_citation(): xml_text = EXAMPLE_BIBTEX_CITATION citation_elem = parse_xml_string(xml_text) - citation = parse_citation(citation_elem, None) + citation_model = parse_citation_elem(citation_elem) + assert citation_model + citation = parse_citation(citation_model, None) + assert citation bibtex = citation.to_bibtex() assert "title={Galaxy" in bibtex From d61937770162a1806b265930fe0a3e47fd6ea0c0 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 28 Jun 2024 08:22:20 -0400 Subject: [PATCH 06/12] Small improvement to typing in metadata_generator. --- .../tool_shed/metadata/metadata_generator.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_shed/metadata/metadata_generator.py b/lib/galaxy/tool_shed/metadata/metadata_generator.py index 4c439ada7a99..90d374bec726 100644 --- a/lib/galaxy/tool_shed/metadata/metadata_generator.py +++ b/lib/galaxy/tool_shed/metadata/metadata_generator.py @@ -12,7 +12,10 @@ Union, ) -from typing_extensions import Protocol +from typing_extensions import ( + Protocol, + TypedDict, +) from galaxy import util from galaxy.model.tool_shed_install import ToolShedRepository @@ -64,6 +67,21 @@ ] +class RepositoryMetadataToolDict(TypedDict): + id: str + guid: str + name: str + version: str + profile: str + description: Optional[str] + version_string_cmd: Optional[str] + tool_config: str + tool_type: str + requirements: Optional[Any] + tests: Optional[Any] + add_to_tool_panel: bool + + class RepositoryProtocol(Protocol): name: str id: str @@ -597,7 +615,7 @@ def generate_tool_metadata(self, tool_config, tool, metadata_dict): # should not be displayed in the tool panel are datatypes converters and DataManager tools # (which are of type 'manage_data'). add_to_tool_panel_attribute = self._set_add_to_tool_panel_attribute_for_tool(tool) - tool_dict = dict( + tool_dict = RepositoryMetadataToolDict( id=tool.id, guid=guid, name=tool.name, From 5359f8ba74687fc07369ab90f2ae5c2e218b7dc8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 28 Jun 2024 05:27:11 -0400 Subject: [PATCH 07/12] Input parameter schema. --- .gitignore | 1 + doc/source/dev/image.Makefile | 11 + doc/source/dev/plantuml_options.txt | 51 + doc/source/dev/plantuml_style.txt | 9 + doc/source/dev/tool_state_api.plantuml.txt | 17 + .../dev/tool_state_state_classes.plantuml.txt | 41 + .../schemas/tool_shed_config_schema.yml | 7 + lib/galaxy/tool_util/cwl/parser.py | 6 +- lib/galaxy/tool_util/parameters/__init__.py | 101 ++ lib/galaxy/tool_util/parameters/_types.py | 43 + lib/galaxy/tool_util/parameters/convert.py | 73 ++ lib/galaxy/tool_util/parameters/factory.py | 294 ++++++ lib/galaxy/tool_util/parameters/json.py | 27 + lib/galaxy/tool_util/parameters/models.py | 931 ++++++++++++++++++ lib/galaxy/tool_util/parameters/state.py | 98 ++ lib/galaxy/tool_util/parameters/visitor.py | 56 ++ lib/galaxy/tool_util/parser/cwl.py | 27 +- lib/galaxy/tool_util/parser/xml.py | 3 +- lib/galaxy/tool_util/parser/yaml.py | 3 +- .../tool_util/unittest_utils/parameters.py | 46 + lib/tool_shed/managers/tool_state_cache.py | 42 + lib/tool_shed/managers/tools.py | 93 +- lib/tool_shed/managers/trs.py | 10 +- lib/tool_shed/structured_app.py | 2 + .../test/functional/test_shed_tools.py | 12 +- lib/tool_shed/webapp/api2/tools.py | 51 +- lib/tool_shed/webapp/app.py | 2 + lib/tool_shed/webapp/model/__init__.py | 2 + .../tools/parameters/cwl_boolean.cwl | 15 + .../tools/parameters/cwl_boolean_optional.cwl | 15 + .../tools/parameters/cwl_directory.cwl | 15 + test/functional/tools/parameters/cwl_file.cwl | 15 + .../functional/tools/parameters/cwl_float.cwl | 15 + .../tools/parameters/cwl_float_optional.cwl | 15 + test/functional/tools/parameters/cwl_int.cwl | 15 + .../tools/parameters/cwl_int_optional.cwl | 15 + .../tools/parameters/cwl_string.cwl | 15 + .../tools/parameters/cwl_string_optional.cwl | 15 + .../tools/parameters/gx_boolean.xml | 29 + .../tools/parameters/gx_boolean_optional.xml | 82 ++ test/functional/tools/parameters/gx_color.xml | 21 + .../parameters/gx_conditional_boolean.xml | 101 ++ .../gx_conditional_boolean_checked.xml | 53 + ...l_boolean_discriminate_on_string_value.xml | 113 +++ .../gx_conditional_boolean_optional.xml | 79 ++ .../gx_conditional_conditional_boolean.xml | 30 + test/functional/tools/parameters/gx_data.xml | 13 + .../tools/parameters/gx_data_collection.xml | 14 + .../gx_data_collection_optional.xml | 14 + .../tools/parameters/gx_data_multiple.xml | 13 + .../parameters/gx_data_multiple_optional.xml | 13 + .../tools/parameters/gx_data_optional.xml | 13 + test/functional/tools/parameters/gx_float.xml | 29 + .../tools/parameters/gx_float_optional.xml | 29 + .../functional/tools/parameters/gx_hidden.xml | 21 + .../tools/parameters/gx_hidden_optional.xml | 21 + test/functional/tools/parameters/gx_int.xml | 29 + .../tools/parameters/gx_int_optional.xml | 21 + .../tools/parameters/gx_repeat_boolean.xml | 15 + .../parameters/gx_repeat_boolean_min.xml | 15 + .../tools/parameters/gx_repeat_data.xml | 21 + .../tools/parameters/gx_repeat_data_min.xml | 24 + .../tools/parameters/gx_section_boolean.xml | 35 + .../tools/parameters/gx_section_data.xml | 21 + .../functional/tools/parameters/gx_select.xml | 27 + .../tools/parameters/gx_select_multiple.xml | 27 + .../gx_select_multiple_optional.xml | 27 + .../tools/parameters/gx_select_optional.xml | 27 + test/functional/tools/parameters/gx_text.xml | 21 + .../tools/parameters/gx_text_optional.xml | 21 + test/functional/tools/parameters/macros.xml | 33 + test/functional/tools/sample_tool_conf.xml | 2 + test/unit/tool_shed/_util.py | 2 + test/unit/tool_shed/test_tool_source.py | 33 + .../tool_util/parameter_specification.yml | 681 +++++++++++++ .../tool_util/test_parameter_specification.py | 226 +++++ 76 files changed, 4193 insertions(+), 12 deletions(-) create mode 100644 doc/source/dev/image.Makefile create mode 100644 doc/source/dev/plantuml_options.txt create mode 100644 doc/source/dev/plantuml_style.txt create mode 100644 doc/source/dev/tool_state_api.plantuml.txt create mode 100644 doc/source/dev/tool_state_state_classes.plantuml.txt create mode 100644 lib/galaxy/tool_util/parameters/__init__.py create mode 100644 lib/galaxy/tool_util/parameters/_types.py create mode 100644 lib/galaxy/tool_util/parameters/convert.py create mode 100644 lib/galaxy/tool_util/parameters/factory.py create mode 100644 lib/galaxy/tool_util/parameters/json.py create mode 100644 lib/galaxy/tool_util/parameters/models.py create mode 100644 lib/galaxy/tool_util/parameters/state.py create mode 100644 lib/galaxy/tool_util/parameters/visitor.py create mode 100644 lib/galaxy/tool_util/unittest_utils/parameters.py create mode 100644 lib/tool_shed/managers/tool_state_cache.py create mode 100644 test/functional/tools/parameters/cwl_boolean.cwl create mode 100644 test/functional/tools/parameters/cwl_boolean_optional.cwl create mode 100644 test/functional/tools/parameters/cwl_directory.cwl create mode 100644 test/functional/tools/parameters/cwl_file.cwl create mode 100644 test/functional/tools/parameters/cwl_float.cwl create mode 100644 test/functional/tools/parameters/cwl_float_optional.cwl create mode 100644 test/functional/tools/parameters/cwl_int.cwl create mode 100644 test/functional/tools/parameters/cwl_int_optional.cwl create mode 100644 test/functional/tools/parameters/cwl_string.cwl create mode 100644 test/functional/tools/parameters/cwl_string_optional.cwl create mode 100644 test/functional/tools/parameters/gx_boolean.xml create mode 100644 test/functional/tools/parameters/gx_boolean_optional.xml create mode 100644 test/functional/tools/parameters/gx_color.xml create mode 100644 test/functional/tools/parameters/gx_conditional_boolean.xml create mode 100644 test/functional/tools/parameters/gx_conditional_boolean_checked.xml create mode 100644 test/functional/tools/parameters/gx_conditional_boolean_discriminate_on_string_value.xml create mode 100644 test/functional/tools/parameters/gx_conditional_boolean_optional.xml create mode 100644 test/functional/tools/parameters/gx_conditional_conditional_boolean.xml create mode 100644 test/functional/tools/parameters/gx_data.xml create mode 100644 test/functional/tools/parameters/gx_data_collection.xml create mode 100644 test/functional/tools/parameters/gx_data_collection_optional.xml create mode 100644 test/functional/tools/parameters/gx_data_multiple.xml create mode 100644 test/functional/tools/parameters/gx_data_multiple_optional.xml create mode 100644 test/functional/tools/parameters/gx_data_optional.xml create mode 100644 test/functional/tools/parameters/gx_float.xml create mode 100644 test/functional/tools/parameters/gx_float_optional.xml create mode 100644 test/functional/tools/parameters/gx_hidden.xml create mode 100644 test/functional/tools/parameters/gx_hidden_optional.xml create mode 100644 test/functional/tools/parameters/gx_int.xml create mode 100644 test/functional/tools/parameters/gx_int_optional.xml create mode 100644 test/functional/tools/parameters/gx_repeat_boolean.xml create mode 100644 test/functional/tools/parameters/gx_repeat_boolean_min.xml create mode 100644 test/functional/tools/parameters/gx_repeat_data.xml create mode 100644 test/functional/tools/parameters/gx_repeat_data_min.xml create mode 100644 test/functional/tools/parameters/gx_section_boolean.xml create mode 100644 test/functional/tools/parameters/gx_section_data.xml create mode 100644 test/functional/tools/parameters/gx_select.xml create mode 100644 test/functional/tools/parameters/gx_select_multiple.xml create mode 100644 test/functional/tools/parameters/gx_select_multiple_optional.xml create mode 100644 test/functional/tools/parameters/gx_select_optional.xml create mode 100644 test/functional/tools/parameters/gx_text.xml create mode 100644 test/functional/tools/parameters/gx_text_optional.xml create mode 100644 test/functional/tools/parameters/macros.xml create mode 100644 test/unit/tool_shed/test_tool_source.py create mode 100644 test/unit/tool_util/parameter_specification.yml create mode 100644 test/unit/tool_util/test_parameter_specification.py diff --git a/.gitignore b/.gitignore index 1a6b7d7f2a07..c907531198ef 100644 --- a/.gitignore +++ b/.gitignore @@ -153,6 +153,7 @@ doc/build doc/schema.md doc/source/admin/config_logging_default_yaml.rst doc/source/dev/schema.md +doc/source/dev/plantuml.jar client/docs/dist # Webpack stats diff --git a/doc/source/dev/image.Makefile b/doc/source/dev/image.Makefile new file mode 100644 index 000000000000..75cb5e1910e4 --- /dev/null +++ b/doc/source/dev/image.Makefile @@ -0,0 +1,11 @@ +MINDMAPS := $(wildcard *.mindmap.yml) +INPUTS := $(wildcard *.plantuml.txt) +OUTPUTS := $(INPUTS:.txt=.svg) + +all: plantuml.jar $(MINDMAPS) $(OUTPUTS) + +$(OUTPUTS): $(INPUTS) $(MINDMAPS) + java -jar plantuml.jar -c plantuml_options.txt -tsvg $(INPUTS) + +plantuml.jar: + wget http://jaist.dl.sourceforge.net/project/plantuml/plantuml.jar || curl --output plantuml.jar http://jaist.dl.sourceforge.net/project/plantuml/plantuml.jar diff --git a/doc/source/dev/plantuml_options.txt b/doc/source/dev/plantuml_options.txt new file mode 100644 index 000000000000..70424ef26736 --- /dev/null +++ b/doc/source/dev/plantuml_options.txt @@ -0,0 +1,51 @@ +' skinparam handwritten true +' skinparam roundcorner 20 + +skinparam class { + ArrowFontColor DarkOrange + BackgroundColor #FFEFD5 + ArrowColor Orange + BorderColor DarkOrange +} + +skinparam object { + ArrowFontColor DarkOrange + BackgroundColor #FFEFD5 + BackgroundColor #FFEFD5 + ArrowColor Orange + BorderColor DarkOrange +} + +skinparam ComponentBackgroundColor #FFEFD5 +skinparam ComponentBorderColor DarkOrange + +skinparam DatabaseBackgroundColor #FFEFD5 +skinparam DatabaseBorderColor DarkOrange + +skinparam StorageBackgroundColor #FFEFD5 +skinparam StorageBorderColor DarkOrange + +skinparam QueueBackgroundColor #FFEFD5 +skinparam QueueBorderColor DarkOrange + +skinparam note { + BackgroundColor #FFEFD5 + BorderColor #BF5700 +} + +skinparam sequence { + ArrowColor Orange + ArrowFontColor DarkOrange + ActorBorderColor DarkOrange + ActorBackgroundColor #FFEFD5 + + ParticipantBorderColor DarkOrange + ParticipantBackgroundColor #FFEFD5 + + LifeLineBorderColor DarkOrange + LifeLineBackgroundColor #FFEFD5 + + DividerBorderColor DarkOrange + GroupBorderColor DarkOrange +} + diff --git a/doc/source/dev/plantuml_style.txt b/doc/source/dev/plantuml_style.txt new file mode 100644 index 000000000000..18911d622b75 --- /dev/null +++ b/doc/source/dev/plantuml_style.txt @@ -0,0 +1,9 @@ + diff --git a/doc/source/dev/tool_state_api.plantuml.txt b/doc/source/dev/tool_state_api.plantuml.txt new file mode 100644 index 000000000000..060b4c5bdb9e --- /dev/null +++ b/doc/source/dev/tool_state_api.plantuml.txt @@ -0,0 +1,17 @@ +@startuml +'!include plantuml_options.txt +participant "API Request" as apireq +boundary "Jobs API" as api +participant "Job Service" as service +database Database as database +queue TaskQueue as queue +apireq -> api : HTTP JSON +api -> service : To boundary +service -> service : Build RequestToolState +service -> service : Validate RequestToolState (pydantic) +service -> service : decode() RequestToolState \ninto RequestInternalToolState +service -> database : Serialize RequestInternalToolState +service -> queue : Queue QueueJobs with reference to\npersisted RequestInternalToolState +service -> api : JobCreateResponse\n (pydantic model) +api -> apireq : JobCreateResponse\n (as json) +@enduml diff --git a/doc/source/dev/tool_state_state_classes.plantuml.txt b/doc/source/dev/tool_state_state_classes.plantuml.txt new file mode 100644 index 000000000000..612c13d8e683 --- /dev/null +++ b/doc/source/dev/tool_state_state_classes.plantuml.txt @@ -0,0 +1,41 @@ +@startuml +!include plantuml_options.txt + +package galaxy.tool_util.parameters.state { + +class ToolState { +state_representation: str +input_state: Dict[str, Any] ++ validate(input_models: ToolParameterBundle) ++ {abstract} _to_base_model(input_models: ToolParameterBundle): Optional[Type[BaseModel]] +} + +class RequestToolState { +state_representation = "request" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Object references of the form \n{src: "hda", id: }.\n Allow mapping/reduce constructs. + +class RequestInternalToolState { +state_representation = "request_internal" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] +} +note bottom: Object references of the form \n{src: "hda", id: }.\n Allow mapping/reduce constructs. + +class JobInternalToolState { +state_representation = "job_internal" ++ _to_base_model(input_models: ToolParameterBundle): Type[BaseModel] + +} +note bottom: Object references of the form \n{src: "hda", id: }.\n Mapping constructs expanded out.\n (Defaults are inserted?) + +ToolState <|-- RequestToolState +ToolState <|-- RequestInternalToolState +ToolState <|-- JobInternalToolState + +RequestToolState - RequestInternalToolState : decode > + +RequestInternalToolState o-- JobInternalToolState : expand > + +} +@enduml \ No newline at end of file diff --git a/lib/galaxy/config/schemas/tool_shed_config_schema.yml b/lib/galaxy/config/schemas/tool_shed_config_schema.yml index 42ea164d9d5f..2a1eee2b533f 100644 --- a/lib/galaxy/config/schemas/tool_shed_config_schema.yml +++ b/lib/galaxy/config/schemas/tool_shed_config_schema.yml @@ -102,6 +102,13 @@ mapping: the repositories and tools within the Tool Shed given that you specify the following two config options. + tool_state_cache_dir: + type: str + default: database/tool_state_cache + required: false + desc: | + Cache directory for tool state. + repo_name_boost: type: float default: 0.9 diff --git a/lib/galaxy/tool_util/cwl/parser.py b/lib/galaxy/tool_util/cwl/parser.py index 1599955eb796..e4a4ff83fb20 100644 --- a/lib/galaxy/tool_util/cwl/parser.py +++ b/lib/galaxy/tool_util/cwl/parser.py @@ -144,6 +144,10 @@ def galaxy_id(self) -> str: tool_id = tool_id[1:] return tool_id + @abstractmethod + def input_fields(self) -> list: + """Return InputInstance objects describing mapping to Galaxy inputs.""" + @abstractmethod def input_instances(self): """Return InputInstance objects describing mapping to Galaxy inputs.""" @@ -236,7 +240,7 @@ def label(self): else: return "" - def input_fields(self): + def input_fields(self) -> list: input_records_schema = self._eval_schema(self._tool.inputs_record_schema) if input_records_schema["type"] != "record": raise Exception("Unhandled CWL tool input structure") diff --git a/lib/galaxy/tool_util/parameters/__init__.py b/lib/galaxy/tool_util/parameters/__init__.py new file mode 100644 index 000000000000..4c50680fa118 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/__init__.py @@ -0,0 +1,101 @@ +from .convert import ( + decode, + encode, +) +from .factory import ( + from_input_source, + input_models_for_pages, + input_models_for_tool_source, + input_models_from_json, + tool_parameter_bundle_from_json, +) +from .json import to_json_schema_string +from .models import ( + BooleanParameterModel, + ColorParameterModel, + ConditionalParameterModel, + ConditionalWhen, + CwlBooleanParameterModel, + CwlDirectoryParameterModel, + CwlFileParameterModel, + CwlFloatParameterModel, + CwlIntegerParameterModel, + CwlNullParameterModel, + CwlStringParameterModel, + CwlUnionParameterModel, + DataCollectionParameterModel, + DataParameterModel, + FloatParameterModel, + HiddenParameterModel, + IntegerParameterModel, + LabelValue, + RepeatParameterModel, + RulesParameterModel, + SelectParameterModel, + TextParameterModel, + ToolParameterBundle, + ToolParameterBundleModel, + ToolParameterModel, + ToolParameterT, + validate_against_model, + validate_internal_job, + validate_internal_request, + validate_request, + validate_test_case, +) +from .state import ( + JobInternalToolState, + RequestInternalToolState, + RequestToolState, + TestCaseToolState, + ToolState, +) +from .visitor import visit_input_values + +__all__ = ( + "from_input_source", + "input_models_for_pages", + "input_models_for_tool_source", + "tool_parameter_bundle_from_json", + "input_models_from_json", + "JobInternalToolState", + "ToolParameterBundle", + "ToolParameterBundleModel", + "ToolParameterModel", + "IntegerParameterModel", + "BooleanParameterModel", + "CwlFileParameterModel", + "CwlFloatParameterModel", + "CwlIntegerParameterModel", + "CwlStringParameterModel", + "CwlNullParameterModel", + "CwlUnionParameterModel", + "CwlBooleanParameterModel", + "CwlDirectoryParameterModel", + "TextParameterModel", + "FloatParameterModel", + "HiddenParameterModel", + "ColorParameterModel", + "RulesParameterModel", + "DataParameterModel", + "DataCollectionParameterModel", + "LabelValue", + "SelectParameterModel", + "ConditionalParameterModel", + "ConditionalWhen", + "RepeatParameterModel", + "validate_against_model", + "validate_internal_job", + "validate_internal_request", + "validate_request", + "validate_test_case", + "ToolState", + "TestCaseToolState", + "ToolParameterT", + "to_json_schema_string", + "RequestToolState", + "RequestInternalToolState", + "visit_input_values", + "decode", + "encode", +) diff --git a/lib/galaxy/tool_util/parameters/_types.py b/lib/galaxy/tool_util/parameters/_types.py new file mode 100644 index 000000000000..4a33f6406a50 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/_types.py @@ -0,0 +1,43 @@ +"""Type utilities for building pydantic models for tool parameters. + +Lots of mypy exceptions in here - this code is all well tested and the exceptions +are fine otherwise because we're using the typing system to interact with pydantic +and build runtime models not to use mypy to type check static code. +""" + +from typing import ( + cast, + List, + Optional, + Type, + Union, +) + +# https://stackoverflow.com/questions/56832881/check-if-a-field-is-typing-optional +from typing_extensions import ( + get_args, + get_origin, +) + + +def optional_if_needed(type: Type, is_optional: bool) -> Type: + return_type: Type = type + if is_optional: + return_type = Optional[type] # type: ignore[assignment] + return return_type + + +def union_type(args: List[Type]) -> Type: + return Union[tuple(args)] # type: ignore[return-value] + + +def list_type(arg: Type) -> Type: + return List[arg] # type: ignore[valid-type] + + +def cast_as_type(arg) -> Type: + return cast(Type, arg) + + +def is_optional(field) -> bool: + return get_origin(field) is Union and type(None) in get_args(field) diff --git a/lib/galaxy/tool_util/parameters/convert.py b/lib/galaxy/tool_util/parameters/convert.py new file mode 100644 index 000000000000..14caed47e92c --- /dev/null +++ b/lib/galaxy/tool_util/parameters/convert.py @@ -0,0 +1,73 @@ +"""Utilities for converting between request states. +""" + +from typing import ( + Any, + Callable, +) + +from .models import ( + ToolParameterBundle, + ToolParameterT, +) +from .state import ( + RequestInternalToolState, + RequestToolState, +) +from .visitor import ( + visit_input_values, + VISITOR_NO_REPLACEMENT, +) + + +def decode( + external_state: RequestToolState, input_models: ToolParameterBundle, decode_id: Callable[[str], int] +) -> RequestInternalToolState: + """Prepare an external representation of tool state (request) for storing in the database (request_internal).""" + + external_state.validate(input_models) + + def decode_callback(parameter: ToolParameterT, value: Any): + if parameter.parameter_type == "gx_data": + assert isinstance(value, dict), str(value) + assert "id" in value + decoded_dict = value.copy() + decoded_dict["id"] = decode_id(value["id"]) + return decoded_dict + else: + return VISITOR_NO_REPLACEMENT + + internal_state_dict = visit_input_values( + input_models, + external_state, + decode_callback, + ) + + internal_request_state = RequestInternalToolState(internal_state_dict) + internal_request_state.validate(input_models) + return internal_request_state + + +def encode( + external_state: RequestInternalToolState, input_models: ToolParameterBundle, encode_id: Callable[[int], str] +) -> RequestToolState: + """Prepare an external representation of tool state (request) for storing in the database (request_internal).""" + + def encode_callback(parameter: ToolParameterT, value: Any): + if parameter.parameter_type == "gx_data": + assert isinstance(value, dict), str(value) + assert "id" in value + encoded_dict = value.copy() + encoded_dict["id"] = encode_id(value["id"]) + return encoded_dict + else: + return VISITOR_NO_REPLACEMENT + + request_state_dict = visit_input_values( + input_models, + external_state, + encode_callback, + ) + request_state = RequestToolState(request_state_dict) + request_state.validate(input_models) + return request_state diff --git a/lib/galaxy/tool_util/parameters/factory.py b/lib/galaxy/tool_util/parameters/factory.py new file mode 100644 index 000000000000..0b28e7da3288 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/factory.py @@ -0,0 +1,294 @@ +from typing import ( + Any, + Dict, + List, + Optional, +) + +from galaxy.tool_util.parser.cwl import CwlInputSource +from galaxy.tool_util.parser.interface import ( + InputSource, + PageSource, + PagesSource, + ToolSource, +) +from galaxy.util import string_as_bool +from .models import ( + BooleanParameterModel, + ColorParameterModel, + ConditionalParameterModel, + ConditionalWhen, + CwlBooleanParameterModel, + CwlDirectoryParameterModel, + CwlFileParameterModel, + CwlFloatParameterModel, + CwlIntegerParameterModel, + CwlNullParameterModel, + CwlStringParameterModel, + CwlUnionParameterModel, + DataCollectionParameterModel, + DataParameterModel, + FloatParameterModel, + HiddenParameterModel, + IntegerParameterModel, + LabelValue, + RepeatParameterModel, + RulesParameterModel, + SectionParameterModel, + SelectParameterModel, + TextParameterModel, + ToolParameterBundle, + ToolParameterBundleModel, + ToolParameterT, +) + + +class ParameterDefinitionError(Exception): + pass + + +def get_color_value(input_source: InputSource) -> str: + return input_source.get("value", "#000000") + + +def _from_input_source_galaxy(input_source: InputSource) -> ToolParameterT: + input_type = input_source.parse_input_type() + if input_type == "param": + param_type = input_source.get("type") + if param_type == "integer": + optional = input_source.parse_optional() + value = input_source.get("value") + int_value: Optional[int] + if value: + int_value = int(value) + elif optional: + int_value = None + else: + raise ParameterDefinitionError() + return IntegerParameterModel(name=input_source.parse_name(), optional=optional, value=int_value) + elif param_type == "boolean": + nullable = input_source.parse_optional() + checked = input_source.get_bool("checked", None if nullable else False) + return BooleanParameterModel( + name=input_source.parse_name(), + optional=nullable, + value=checked, + ) + elif param_type == "text": + optional = input_source.parse_optional() + return TextParameterModel( + name=input_source.parse_name(), + optional=optional, + ) + elif param_type == "float": + optional = input_source.parse_optional() + value = input_source.get("value") + float_value: Optional[float] + if value: + float_value = float(value) + elif optional: + float_value = None + else: + raise ParameterDefinitionError() + return FloatParameterModel( + name=input_source.parse_name(), + optional=optional, + value=float_value, + ) + elif param_type == "hidden": + optional = input_source.parse_optional() + return HiddenParameterModel( + name=input_source.parse_name(), + optional=optional, + ) + elif param_type == "color": + optional = input_source.parse_optional() + return ColorParameterModel( + name=input_source.parse_name(), + optional=optional, + value=get_color_value(input_source), + ) + elif param_type == "rules": + return RulesParameterModel( + name=input_source.parse_name(), + ) + elif param_type == "data": + optional = input_source.parse_optional() + multiple = input_source.get_bool("multiple", False) + return DataParameterModel( + name=input_source.parse_name(), + optional=optional, + multiple=multiple, + ) + elif param_type == "data_collection": + optional = input_source.parse_optional() + return DataCollectionParameterModel( + name=input_source.parse_name(), + optional=optional, + ) + elif param_type == "select": + # Function... example in devteam cummeRbund. + optional = input_source.parse_optional() + dynamic_options = input_source.get("dynamic_options", None) + dynamic_options_elem = input_source.parse_dynamic_options_elem() + multiple = input_source.get_bool("multiple", False) + is_static = dynamic_options is None and dynamic_options_elem is None + options: Optional[List[LabelValue]] = None + if is_static: + options = [] + for option_label, option_value, selected in input_source.parse_static_options(): + options.append(LabelValue(label=option_label, value=option_value, selected=selected)) + return SelectParameterModel( + name=input_source.parse_name(), + optional=optional, + options=options, + multiple=multiple, + ) + else: + raise Exception(f"Unknown Galaxy parameter type {param_type}") + elif input_type == "conditional": + test_param_input_source = input_source.parse_test_input_source() + test_parameter = _from_input_source_galaxy(test_param_input_source) + whens = [] + default_value = object() + if isinstance(test_parameter, BooleanParameterModel): + default_value = test_parameter.value + # TODO: handle select parameter model... + for value, case_inputs_sources in input_source.parse_when_input_sources(): + if isinstance(test_parameter, BooleanParameterModel): + # TODO: investigate truevalue/falsevalue when... + typed_value = string_as_bool(value) + else: + typed_value = value + + tool_parameter_models = input_models_for_page(case_inputs_sources) + is_default_when = False + if typed_value == default_value: + is_default_when = True + whens.append( + ConditionalWhen(discriminator=value, parameters=tool_parameter_models, is_default_when=is_default_when) + ) + return ConditionalParameterModel( + name=input_source.parse_name(), + test_parameter=test_parameter, + whens=whens, + ) + elif input_type == "repeat": + name = input_source.get("name") + # title = input_source.get("title") + # help = input_source.get("help", None) + instance_sources = input_source.parse_nested_inputs_source() + instance_tool_parameter_models = input_models_for_page(instance_sources) + min_raw = input_source.get("min", None) + max_raw = input_source.get("max", None) + min = int(min_raw) if min_raw is not None else None + max = int(max_raw) if max_raw is not None else None + return RepeatParameterModel( + name=name, + parameters=instance_tool_parameter_models, + min=min, + max=max, + ) + elif input_type == "section": + name = input_source.get("name") + instance_sources = input_source.parse_nested_inputs_source() + instance_tool_parameter_models = input_models_for_page(instance_sources) + return SectionParameterModel( + name=name, + parameters=instance_tool_parameter_models, + ) + else: + raise Exception( + f"Cannot generate tool parameter model for supplied tool source - unknown input_type {input_type}" + ) + + +def _simple_cwl_type_to_model(simple_type: str, input_source: CwlInputSource): + if simple_type == "int": + return CwlIntegerParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "float": + return CwlFloatParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "null": + return CwlNullParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "string": + return CwlStringParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "boolean": + return CwlBooleanParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "org.w3id.cwl.cwl.File": + return CwlFileParameterModel( + name=input_source.parse_name(), + ) + elif simple_type == "org.w3id.cwl.cwl.Directory": + return CwlDirectoryParameterModel( + name=input_source.parse_name(), + ) + raise NotImplementedError( + f"Cannot generate tool parameter model for this CWL artifact yet - contains unknown type {simple_type}." + ) + + +def _from_input_source_cwl(input_source: CwlInputSource) -> ToolParameterT: + schema_salad_field = input_source.field + if schema_salad_field is None: + raise NotImplementedError("Cannot generate tool parameter model for this CWL artifact yet.") + if "type" not in schema_salad_field: + raise NotImplementedError("Cannot generate tool parameter model for this CWL artifact yet.") + schema_salad_type = schema_salad_field["type"] + if isinstance(schema_salad_type, str): + return _simple_cwl_type_to_model(schema_salad_type, input_source) + elif isinstance(schema_salad_type, list): + return CwlUnionParameterModel( + name=input_source.parse_name(), + parameters=[_simple_cwl_type_to_model(t, input_source) for t in schema_salad_type], + ) + else: + raise NotImplementedError("Cannot generate tool parameter model for this CWL artifact yet.") + + +def input_models_from_json(json: List[Dict[str, Any]]) -> ToolParameterBundle: + return ToolParameterBundleModel(input_models=json) + + +def tool_parameter_bundle_from_json(json: Dict[str, Any]) -> ToolParameterBundleModel: + return ToolParameterBundleModel(**json) + + +def input_models_for_tool_source(tool_source: ToolSource) -> ToolParameterBundleModel: + pages = tool_source.parse_input_pages() + return ToolParameterBundleModel(input_models=input_models_for_pages(pages)) + + +def input_models_for_pages(pages: PagesSource) -> List[ToolParameterT]: + input_models = [] + if pages.inputs_defined: + for page_source in pages.page_sources: + input_models.extend(input_models_for_page(page_source)) + + return input_models + + +def input_models_for_page(page_source: PageSource) -> List[ToolParameterT]: + input_models = [] + for input_source in page_source.parse_input_sources(): + tool_parameter_model = from_input_source(input_source) + input_models.append(tool_parameter_model) + return input_models + + +def from_input_source(input_source: InputSource) -> ToolParameterT: + tool_parameter: ToolParameterT + if isinstance(input_source, CwlInputSource): + tool_parameter = _from_input_source_cwl(input_source) + else: + tool_parameter = _from_input_source_galaxy(input_source) + return tool_parameter diff --git a/lib/galaxy/tool_util/parameters/json.py b/lib/galaxy/tool_util/parameters/json.py new file mode 100644 index 000000000000..6796353a4fb7 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/json.py @@ -0,0 +1,27 @@ +import json +from typing import ( + Any, + Dict, +) + +from pydantic.json_schema import GenerateJsonSchema +from typing_extensions import Literal + +MODE = Literal["validation", "serialization"] +DEFAULT_JSON_SCHEMA_MODE: MODE = "validation" + + +class CustomGenerateJsonSchema(GenerateJsonSchema): + + def generate(self, schema, mode: MODE = DEFAULT_JSON_SCHEMA_MODE): + json_schema = super().generate(schema, mode=mode) + json_schema["$schema"] = self.schema_dialect + return json_schema + + +def to_json_schema(model, mode: MODE = DEFAULT_JSON_SCHEMA_MODE) -> Dict[str, Any]: + return model.model_json_schema(schema_generator=CustomGenerateJsonSchema, mode=mode) + + +def to_json_schema_string(model, mode: MODE = DEFAULT_JSON_SCHEMA_MODE) -> str: + return json.dumps(to_json_schema(model, mode=mode), indent=4) diff --git a/lib/galaxy/tool_util/parameters/models.py b/lib/galaxy/tool_util/parameters/models.py new file mode 100644 index 000000000000..49635c4e7187 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/models.py @@ -0,0 +1,931 @@ +# attempt to model requires_value... +# conditional can descend... +from abc import abstractmethod +from typing import ( + Any, + Callable, + cast, + Dict, + Iterable, + List, + Mapping, + NamedTuple, + Optional, + Type, + Union, +) + +from pydantic import ( + BaseModel, + ConfigDict, + create_model, + Discriminator, + Field, + field_validator, + RootModel, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + Tag, + ValidationError, +) +from typing_extensions import ( + Annotated, + Literal, + Protocol, +) + +from galaxy.exceptions import RequestParameterInvalidException +from ._types import ( + cast_as_type, + is_optional, + list_type, + optional_if_needed, + union_type, +) + +# TODO: +# - implement job vs request... +# - drill down +# - implement data_ref on rules and implement some cross model validation +# - Optional conditionals... work through that? +# - Sections - fight that battle again... + +# + request: Return info needed to build request pydantic model at runtime. +# + request_internal: This is a pydantic model to validate what Galaxy expects to find in the database, +# in particular dataset and collection references should be decoded integers. +StateRepresentationT = Literal["request", "request_internal", "job_internal", "test_case"] + + +# could be made more specific - validators need to be classmethod +ValidatorDictT = Dict[str, Callable] + + +class DynamicModelInformation(NamedTuple): + name: str + definition: tuple + validators: ValidatorDictT + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +def allow_batching(job_template: DynamicModelInformation, batch_type: Optional[Type] = None) -> DynamicModelInformation: + job_py_type: Type = job_template.definition[0] + default_value = job_template.definition[1] + batch_type = batch_type or job_py_type + + class BatchRequest(StrictModel): + meta_class: Literal["Batch"] = Field(..., alias="__class__") + values: List[batch_type] # type: ignore[valid-type] + + request_type = union_type([job_py_type, BatchRequest]) + + return DynamicModelInformation( + job_template.name, + (request_type, default_value), + {}, # should we modify these somehow? + ) + + +class Validators: + def validate_not_none(cls, v): + assert v is not None, "null is an invalid value for attribute" + return v + + +class ParamModel(Protocol): + @property + def name(self) -> str: ... + + @property + def request_requires_value(self) -> bool: + # if this is a non-optional type and no default is defined - an + # input value MUST be specified. + ... + + +def dynamic_model_information_from_py_type(param_model: ParamModel, py_type: Type): + name = param_model.name + initialize = ... if param_model.request_requires_value else None + py_type_is_optional = is_optional(py_type) + if not py_type_is_optional and not param_model.request_requires_value: + validators = {"not_null": field_validator(name)(Validators.validate_not_none)} + else: + validators = {} + + return DynamicModelInformation( + name, + (py_type, initialize), + validators, + ) + + +# We probably need incoming (parameter def) and outgoing (parameter value as transmitted) models, +# where value in the incoming model means "default value" and value in the outgoing model is the actual +# value a user has set. (incoming/outgoing from the client perspective). +class BaseToolParameterModelDefinition(BaseModel): + name: str + parameter_type: str + + @abstractmethod + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + """Return info needed to build Pydantic model at runtime for validation.""" + + +class BaseGalaxyToolParameterModelDefinition(BaseToolParameterModelDefinition): + hidden: bool = False + label: Optional[str] = None + help: Optional[str] = None + argument: Optional[str] = None + is_dynamic: bool = False + optional: bool = False + + +class LabelValue(BaseModel): + label: str + value: str + selected: bool + + +class TextParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_text"] = "gx_text" + area: bool = False + default_value: Optional[str] = Field(default=None, alias="value") + default_options: List[LabelValue] = [] + + @property + def py_type(self) -> Type: + return optional_if_needed(StrictStr, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return False + + +class IntegerParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_integer"] = "gx_integer" + optional: bool + value: Optional[int] = None + min: Optional[int] = None + max: Optional[int] = None + + @property + def py_type(self) -> Type: + return optional_if_needed(StrictInt, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return False + + +class FloatParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_float"] = "gx_float" + value: Optional[float] = None + min: Optional[float] = None + max: Optional[float] = None + + @property + def py_type(self) -> Type: + return optional_if_needed(union_type([StrictInt, StrictFloat]), self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return False + + +DataSrcT = Literal["hda", "ldda"] +MultiDataSrcT = Literal["hda", "ldda", "hdca"] +CollectionStrT = Literal["hdca"] + +TestCaseDataSrcT = Literal["File"] + + +class DataRequest(StrictModel): + src: DataSrcT + id: StrictStr + + +class BatchDataInstance(StrictModel): + src: MultiDataSrcT + id: StrictStr + + +class MultiDataInstance(StrictModel): + src: MultiDataSrcT + id: StrictStr + + +MultiDataRequest: Type = union_type([MultiDataInstance, List[MultiDataInstance]]) + + +class DataRequestInternal(StrictModel): + src: DataSrcT + id: StrictInt + + +class BatchDataInstanceInternal(StrictModel): + src: MultiDataSrcT + id: StrictInt + + +class MultiDataInstanceInternal(StrictModel): + src: MultiDataSrcT + id: StrictInt + + +class DataTestCaseValue(StrictModel): + src: TestCaseDataSrcT + path: str + + +class MultipleDataTestCaseValue(RootModel): + root: List[DataTestCaseValue] + + +MultiDataRequestInternal: Type = union_type([MultiDataInstanceInternal, List[MultiDataInstanceInternal]]) + + +class DataParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_data"] = "gx_data" + extensions: List[str] = ["data"] + multiple: bool = False + min: Optional[int] = None + max: Optional[int] = None + + @property + def py_type(self) -> Type: + base_model: Type + if self.multiple: + base_model = MultiDataRequest + else: + base_model = DataRequest + return optional_if_needed(base_model, self.optional) + + @property + def py_type_internal(self) -> Type: + base_model: Type + if self.multiple: + base_model = MultiDataRequestInternal + else: + base_model = DataRequestInternal + return optional_if_needed(base_model, self.optional) + + @property + def py_type_test_case(self) -> Type: + base_model: Type + if self.multiple: + base_model = MultiDataRequestInternal + else: + base_model = DataTestCaseValue + return optional_if_needed(base_model, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + if state_representation == "request": + return allow_batching(dynamic_model_information_from_py_type(self, self.py_type), BatchDataInstance) + elif state_representation == "request_internal": + return allow_batching( + dynamic_model_information_from_py_type(self, self.py_type_internal), BatchDataInstanceInternal + ) + elif state_representation == "job_internal": + return dynamic_model_information_from_py_type(self, self.py_type_internal) + elif state_representation == "test_case": + return dynamic_model_information_from_py_type(self, self.py_type_test_case) + + @property + def request_requires_value(self) -> bool: + return not self.optional + + +class DataCollectionRequest(StrictModel): + src: CollectionStrT + id: StrictStr + + +class DataCollectionRequestInternal(StrictModel): + src: CollectionStrT + id: StrictInt + + +class DataCollectionParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_data_collection"] = "gx_data_collection" + collection_type: Optional[str] = None + extensions: List[str] = ["data"] + + @property + def py_type(self) -> Type: + return optional_if_needed(DataCollectionRequest, self.optional) + + @property + def py_type_internal(self) -> Type: + return optional_if_needed(DataCollectionRequestInternal, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + if state_representation == "request": + return allow_batching(dynamic_model_information_from_py_type(self, self.py_type)) + elif state_representation == "request_internal": + return allow_batching(dynamic_model_information_from_py_type(self, self.py_type_internal)) + else: + raise NotImplementedError("...") + + @property + def request_requires_value(self) -> bool: + return not self.optional + + +class HiddenParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_hidden"] = "gx_hidden" + + @property + def py_type(self) -> Type: + return optional_if_needed(StrictStr, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return not self.optional + + +def ensure_color_valid(value: Optional[Any]): + if value is None: + return + if not isinstance(value, str): + raise ValueError(f"Invalid color value type {value.__class__} encountered.") + value_str: str = value + message = f"Invalid color value string format {value_str} encountered." + if len(value_str) != 7: + raise ValueError(message + "0") + if value_str[0] != "#": + raise ValueError(message + "1") + for byte_str in value_str[1:]: + if byte_str not in "0123456789abcdef": + raise ValueError(message + "2") + + +class ColorParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_color"] = "gx_color" + value: Optional[str] = None + + @property + def py_type(self) -> Type: + return optional_if_needed(StrictStr, self.optional) + + @staticmethod + def validate_color_str(value) -> str: + ensure_color_valid(value) + return value + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + validators = {"color_format": field_validator(self.name)(ColorParameterModel.validate_color_str)} + return DynamicModelInformation( + self.name, + (self.py_type, ...), + validators, + ) + + @property + def request_requires_value(self) -> bool: + return False + + +class BooleanParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_boolean"] = "gx_boolean" + value: Optional[bool] = False + truevalue: Optional[str] = None + falsevalue: Optional[str] = None + + @property + def py_type(self) -> Type: + return optional_if_needed(StrictBool, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + # these parameters always have an implicit default - either None if + # if it is optional or 'checked' if not (itself defaulting to False). + return False + + +class DirectoryUriParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_directory_uri"] + value: Optional[str] + + @property + def request_requires_value(self) -> bool: + return True + + +class RulesMapping(StrictModel): + type: str + columns: List[StrictInt] + + +class RulesModel(StrictModel): + rules: List[Dict[str, Any]] + mappings: List[RulesMapping] + + +class RulesParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_rules"] = "gx_rules" + + @property + def py_type(self) -> Type: + return RulesModel + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return True + + +class SelectParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_select"] = "gx_select" + options: Optional[List[LabelValue]] = None + multiple: bool + + @property + def py_type(self) -> Type: + if self.options is not None: + literal_options: List[Type] = [cast_as_type(Literal[o.value]) for o in self.options] + py_type = union_type(literal_options) + else: + py_type = StrictStr + if self.multiple: + py_type = list_type(py_type) + return optional_if_needed(py_type, self.optional) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def has_selected_static_option(self): + return self.options is not None and any(o.selected for o in self.options) + + @property + def request_requires_value(self) -> bool: + return not self.optional and not self.has_selected_static_option + + +DiscriminatorType = Union[bool, str] + + +class ConditionalWhen(StrictModel): + discriminator: DiscriminatorType + parameters: List["ToolParameterT"] + is_default_when: bool + + +class ConditionalParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_conditional"] = "gx_conditional" + test_parameter: Union[BooleanParameterModel, SelectParameterModel] + whens: List[ConditionalWhen] + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + test_param_name = self.test_parameter.name + test_info = self.test_parameter.pydantic_template(state_representation) + extra_validators = test_info.validators + test_parameter_requires_value = self.test_parameter.request_requires_value + when_types: List[Type[BaseModel]] = [] + default_type = None + for when in self.whens: + discriminator = when.discriminator + parameters = when.parameters + if test_parameter_requires_value: + initialize_test = ... + else: + initialize_test = None + + extra_kwd = {test_param_name: (Union[str, bool], initialize_test)} + when_types.append( + cast( + Type[BaseModel], + Annotated[ + create_field_model( + parameters, + f"When_{test_param_name}_{discriminator}", + state_representation, + extra_kwd=extra_kwd, + extra_validators=extra_validators, + ), + Tag(str(discriminator)), + ], + ) + ) + if when.is_default_when: + extra_kwd = {} + default_type = create_field_model( + parameters, + f"When_{test_param_name}___absent", + state_representation, + extra_kwd=extra_kwd, + extra_validators={}, + ) + when_types.append(cast(Type[BaseModel], Annotated[default_type, Tag("__absent__")])) + + def model_x_discriminator(v: Any) -> str: + if test_param_name not in v: + return "__absent__" + else: + test_param_val = v[test_param_name] + if test_param_val is True: + return "true" + elif test_param_val is False: + return "false" + else: + return str(test_param_val) + + cond_type = union_type(when_types) + + class ConditionalType(RootModel): + root: cond_type = Field(..., discriminator=Discriminator(model_x_discriminator)) # type: ignore[valid-type] + + if default_type is not None: + initialize_cond = None + else: + initialize_cond = ... + + py_type = ConditionalType + + return DynamicModelInformation( + self.name, + (py_type, initialize_cond), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return False # TODO + + +class RepeatParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_repeat"] = "gx_repeat" + parameters: List["ToolParameterT"] + min: Optional[int] = None + max: Optional[int] = None + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + # Maybe validators for min and max... + instance_class: Type[BaseModel] = create_field_model( + self.parameters, f"Repeat_{self.name}", state_representation + ) + + class RepeatType(RootModel): + root: List[instance_class] = Field(..., min_length=self.min, max_length=self.max) # type: ignore[valid-type] + + return DynamicModelInformation( + self.name, + (RepeatType, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return True # TODO: + + +class SectionParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["gx_section"] = "gx_section" + parameters: List["ToolParameterT"] + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + instance_class: Type[BaseModel] = create_field_model( + self.parameters, f"Section_{self.name}", state_representation + ) + if self.request_requires_value: + initialize_section = ... + else: + initialize_section = None + return DynamicModelInformation( + self.name, + (instance_class, initialize_section), + {}, + ) + + @property + def request_requires_value(self) -> bool: + any_request_parameters_required = False + for parameter in self.parameters: + if parameter.request_requires_value: + any_request_parameters_required = True + break + return any_request_parameters_required + + +LiteralNone: Type = Literal[None] # type: ignore[assignment] + + +class CwlNullParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_null"] = "cwl_null" + + @property + def py_type(self) -> Type: + return LiteralNone + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return False + + +class CwlStringParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_string"] = "cwl_string" + + @property + def py_type(self) -> Type: + return StrictStr + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return True + + +class CwlIntegerParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_integer"] = "cwl_integer" + + @property + def py_type(self) -> Type: + return StrictInt + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return True + + +class CwlFloatParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_float"] = "cwl_float" + + @property + def py_type(self) -> Type: + return union_type([StrictFloat, StrictInt]) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return True + + +class CwlBooleanParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_boolean"] = "cwl_boolean" + + @property + def py_type(self) -> Type: + return StrictBool + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return True + + +class CwlUnionParameterModel(BaseToolParameterModelDefinition): + parameter_type: Literal["cwl_union"] = "cwl_union" + parameters: List["CwlParameterT"] + + @property + def py_type(self) -> Type: + union_of_cwl_types: List[Type] = [] + for parameter in self.parameters: + union_of_cwl_types.append(parameter.py_type) + return union_type(union_of_cwl_types) + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return DynamicModelInformation( + self.name, + (self.py_type, ...), + {}, + ) + + @property + def request_requires_value(self) -> bool: + return False # TODO: + + +class CwlFileParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["cwl_file"] = "cwl_file" + + @property + def py_type(self) -> Type: + return DataRequest + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return True + + +class CwlDirectoryParameterModel(BaseGalaxyToolParameterModelDefinition): + parameter_type: Literal["cwl_directory"] = "cwl_directory" + + @property + def py_type(self) -> Type: + return DataRequest + + def pydantic_template(self, state_representation: StateRepresentationT) -> DynamicModelInformation: + return dynamic_model_information_from_py_type(self, self.py_type) + + @property + def request_requires_value(self) -> bool: + return True + + +CwlParameterT = Union[ + CwlIntegerParameterModel, + CwlFloatParameterModel, + CwlStringParameterModel, + CwlBooleanParameterModel, + CwlNullParameterModel, + CwlFileParameterModel, + CwlDirectoryParameterModel, + CwlUnionParameterModel, +] + +GalaxyParameterT = Union[ + TextParameterModel, + IntegerParameterModel, + FloatParameterModel, + BooleanParameterModel, + HiddenParameterModel, + SelectParameterModel, + DataParameterModel, + DataCollectionParameterModel, + DirectoryUriParameterModel, + RulesParameterModel, + ColorParameterModel, + ConditionalParameterModel, + RepeatParameterModel, + SectionParameterModel, +] + +ToolParameterT = Union[ + CwlParameterT, + GalaxyParameterT, +] + + +class ToolParameterModel(RootModel): + root: ToolParameterT = Field(..., discriminator="parameter_type") + + +ConditionalWhen.model_rebuild() +ConditionalParameterModel.model_rebuild() +RepeatParameterModel.model_rebuild() +CwlUnionParameterModel.model_rebuild() + + +class ToolParameterBundle(Protocol): + """An object having a dictionary of input models (i.e. a 'Tool')""" + + # TODO: rename to parameters to align with ConditionalWhen and Repeat. + input_models: List[ToolParameterT] + + +class ToolParameterBundleModel(BaseModel): + input_models: List[ToolParameterT] + + +def parameters_by_name(tool_parameter_bundle: ToolParameterBundle) -> Dict[str, ToolParameterT]: + as_dict = {} + for input_model in simple_input_models(tool_parameter_bundle.input_models): + as_dict[input_model.name] = input_model + return as_dict + + +def to_simple_model(input_parameter: Union[ToolParameterModel, ToolParameterT]) -> ToolParameterT: + if input_parameter.__class__ == ToolParameterModel: + assert isinstance(input_parameter, ToolParameterModel) + return cast(ToolParameterT, input_parameter.root) + else: + return cast(ToolParameterT, input_parameter) + + +def simple_input_models( + input_models: Union[List[ToolParameterModel], List[ToolParameterT]] +) -> Iterable[ToolParameterT]: + return [to_simple_model(m) for m in input_models] + + +def create_model_strict(*args, **kwd) -> Type[BaseModel]: + model_config = ConfigDict(extra="forbid") + + return create_model(*args, __config__=model_config, **kwd) + + +def create_request_model(tool: ToolParameterBundle, name: str = "DynamicModelForTool") -> Type[BaseModel]: + return create_field_model(tool.input_models, name, "request") + + +def create_request_internal_model(tool: ToolParameterBundle, name: str = "DynamicModelForTool") -> Type[BaseModel]: + return create_field_model(tool.input_models, name, "request_internal") + + +def create_job_internal_model(tool: ToolParameterBundle, name: str = "DynamicModelForTool") -> Type[BaseModel]: + return create_field_model(tool.input_models, name, "job_internal") + + +def create_test_case_model(tool: ToolParameterBundle, name: str = "DynamicModelForTool") -> Type[BaseModel]: + return create_field_model(tool.input_models, name, "test_case") + + +def create_field_model( + tool_parameter_models: Union[List[ToolParameterModel], List[ToolParameterT]], + name: str, + state_representation: StateRepresentationT, + extra_kwd: Optional[Mapping[str, tuple]] = None, + extra_validators: Optional[ValidatorDictT] = None, +) -> Type[BaseModel]: + kwd: Dict[str, tuple] = {} + if extra_kwd: + kwd.update(extra_kwd) + model_validators = (extra_validators or {}).copy() + + for input_model in tool_parameter_models: + input_model = to_simple_model(input_model) + input_name = input_model.name + pydantic_request_template = input_model.pydantic_template(state_representation) + kwd[input_name] = pydantic_request_template.definition + input_validators = pydantic_request_template.validators + for validator_name, validator_callable in input_validators.items(): + model_validators[f"{input_name}_{validator_name}"] = validator_callable + + pydantic_model = create_model_strict(name, __validators__=model_validators, **kwd) + return pydantic_model + + +def validate_against_model(pydantic_model: Type[BaseModel], parameter_state: Dict[str, Any]) -> None: + try: + pydantic_model(**parameter_state) + except ValidationError as e: + # TODO: Improve this or maybe add a handler for this in the FastAPI exception + # handler. + raise RequestParameterInvalidException(str(e)) + + +def validate_request(tool: ToolParameterBundle, request: Dict[str, Any]) -> None: + pydantic_model = create_request_model(tool) + validate_against_model(pydantic_model, request) + + +def validate_internal_request(tool: ToolParameterBundle, request: Dict[str, Any]) -> None: + pydantic_model = create_request_internal_model(tool) + validate_against_model(pydantic_model, request) + + +def validate_internal_job(tool: ToolParameterBundle, request: Dict[str, Any]) -> None: + pydantic_model = create_job_internal_model(tool) + validate_against_model(pydantic_model, request) + + +def validate_test_case(tool: ToolParameterBundle, request: Dict[str, Any]) -> None: + pydantic_model = create_test_case_model(tool) + validate_against_model(pydantic_model, request) diff --git a/lib/galaxy/tool_util/parameters/state.py b/lib/galaxy/tool_util/parameters/state.py new file mode 100644 index 000000000000..3991054bbd33 --- /dev/null +++ b/lib/galaxy/tool_util/parameters/state.py @@ -0,0 +1,98 @@ +from abc import ( + ABC, + abstractmethod, +) +from typing import ( + Any, + Dict, + List, + Optional, + Type, + Union, +) + +from pydantic import BaseModel +from typing_extensions import Literal + +from .models import ( + create_job_internal_model, + create_request_internal_model, + create_request_model, + StateRepresentationT, + ToolParameterBundle, + ToolParameterBundleModel, + ToolParameterT, + validate_against_model, +) + +HasToolParameters = Union[List[ToolParameterT], ToolParameterBundle] + + +class ToolState(ABC): + input_state: Dict[str, Any] + + def __init__(self, input_state: Dict[str, Any]): + self.input_state = input_state + + def _validate(self, pydantic_model: Type[BaseModel]) -> None: + validate_against_model(pydantic_model, self.input_state) + + def validate(self, input_models: HasToolParameters) -> None: + base_model = self.parameter_model_for(input_models) + if base_model is None: + raise NotImplementedError( + f"Validating tool state against state representation {self.state_representation} is not implemented." + ) + self._validate(base_model) + + @property + @abstractmethod + def state_representation(self) -> StateRepresentationT: + """Get state representation of the inputs.""" + + @classmethod + def parameter_model_for(cls, input_models: HasToolParameters) -> Optional[Type[BaseModel]]: + bundle: ToolParameterBundle + if isinstance(input_models, list): + bundle = ToolParameterBundleModel(input_models=input_models) + else: + bundle = input_models + return cls._parameter_model_for(bundle) + + @classmethod + @abstractmethod + def _parameter_model_for(cls, input_models: ToolParameterBundle) -> Optional[Type[BaseModel]]: + """Return a model type for this tool state kind.""" + + +class RequestToolState(ToolState): + state_representation: Literal["request"] = "request" + + @classmethod + def _parameter_model_for(cls, input_models: ToolParameterBundle) -> Type[BaseModel]: + return create_request_model(input_models) + + +class RequestInternalToolState(ToolState): + state_representation: Literal["request_internal"] = "request_internal" + + @classmethod + def _parameter_model_for(cls, input_models: ToolParameterBundle) -> Type[BaseModel]: + return create_request_internal_model(input_models) + + +class JobInternalToolState(ToolState): + state_representation: Literal["job_internal"] = "job_internal" + + @classmethod + def _parameter_model_for(cls, input_models: ToolParameterBundle) -> Type[BaseModel]: + return create_job_internal_model(input_models) + + +class TestCaseToolState(ToolState): + state_representation: Literal["test_case"] = "test_case" + + @classmethod + def _parameter_model_for(cls, input_models: ToolParameterBundle) -> Type[BaseModel]: + # implement a test case model... + return create_request_internal_model(input_models) diff --git a/lib/galaxy/tool_util/parameters/visitor.py b/lib/galaxy/tool_util/parameters/visitor.py new file mode 100644 index 000000000000..7b68afa4a0aa --- /dev/null +++ b/lib/galaxy/tool_util/parameters/visitor.py @@ -0,0 +1,56 @@ +from typing import ( + Any, + Dict, + Iterable, +) + +from typing_extensions import Protocol + +from .models import ( + simple_input_models, + ToolParameterBundle, + ToolParameterT, +) +from .state import ToolState + +VISITOR_NO_REPLACEMENT = object() +VISITOR_UNDEFINED = object() + + +class Callback(Protocol): + def __call__(self, parameter: ToolParameterT, value: Any): + pass + + +def visit_input_values( + input_models: ToolParameterBundle, + tool_state: ToolState, + callback: Callback, + no_replacement_value=VISITOR_NO_REPLACEMENT, +) -> Dict[str, Any]: + return _visit_input_values( + simple_input_models(input_models.input_models), + tool_state.input_state, + callback=callback, + no_replacement_value=no_replacement_value, + ) + + +def _visit_input_values( + input_models: Iterable[ToolParameterT], + input_values: Dict[str, Any], + callback: Callback, + no_replacement_value=VISITOR_NO_REPLACEMENT, +) -> Dict[str, Any]: + new_input_values = {} + for model in input_models: + name = model.name + input_value = input_values.get(name, VISITOR_UNDEFINED) + replacement = callback(model, input_value) + if replacement != no_replacement_value: + new_input_values[name] = replacement + elif replacement is VISITOR_UNDEFINED: + pass + else: + new_input_values[name] = input_value + return new_input_values diff --git a/lib/galaxy/tool_util/parser/cwl.py b/lib/galaxy/tool_util/parser/cwl.py index 1647c39aa701..45a4634ae82f 100644 --- a/lib/galaxy/tool_util/parser/cwl.py +++ b/lib/galaxy/tool_util/parser/cwl.py @@ -179,18 +179,41 @@ def to_string(self): return json.dumps(self.tool_proxy.to_persistent_representation()) +class CwlInputSource(YamlInputSource): + def __init__(self, as_dict, as_field): + super().__init__(as_dict) + self._field = as_field + + @property + def field(self): + return self._field + + class CwlPageSource(PageSource): def __init__(self, tool_proxy): cwl_instances = tool_proxy.input_instances() - self._input_list = list(map(self._to_input_source, cwl_instances)) + input_fields = tool_proxy.input_fields() + input_list = [] + for cwl_instance in cwl_instances: + name = cwl_instance.name + input_field = None + for field in input_fields: + if field["name"] == name: + input_field = field + input_list.append(CwlInputSource(cwl_instance.to_dict(), input_field)) + + self._input_list = input_list def _to_input_source(self, input_instance): as_dict = input_instance.to_dict() - return YamlInputSource(as_dict) + return CwlInputSource(as_dict) def parse_input_sources(self): return self._input_list + def input_fields(self): + return self._input_fields + __all__ = ( "CwlToolSource", diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 6cc850307934..00a8be56df77 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -11,6 +11,7 @@ Iterable, List, Optional, + Tuple, ) from packaging.version import Version @@ -1252,7 +1253,7 @@ def parse_dynamic_options_elem(self): options_elem = self.input_elem.find("options") return options_elem - def parse_static_options(self): + def parse_static_options(self) -> List[Tuple[str, str, bool]]: """ >>> from galaxy.util import parse_xml_string_to_etree >>> xml = '' diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index d6f135c027eb..8e7eaffecb7c 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -4,6 +4,7 @@ Dict, List, Optional, + Tuple, ) import packaging.version @@ -364,7 +365,7 @@ def parse_when_input_sources(self): sources.append((value, case_page_source)) return sources - def parse_static_options(self): + def parse_static_options(self) -> List[Tuple[str, str, bool]]: static_options = [] input_dict = self.input_dict for option in input_dict.get("options", {}): diff --git a/lib/galaxy/tool_util/unittest_utils/parameters.py b/lib/galaxy/tool_util/unittest_utils/parameters.py new file mode 100644 index 000000000000..d3be68b7cca2 --- /dev/null +++ b/lib/galaxy/tool_util/unittest_utils/parameters.py @@ -0,0 +1,46 @@ +import os +from typing import List + +from galaxy.tool_util.parameters import ( + from_input_source, + ToolParameterBundle, + ToolParameterT, +) +from galaxy.tool_util.parser.factory import get_tool_source +from galaxy.util import galaxy_directory + + +class ParameterBundle(ToolParameterBundle): + input_models: List[ToolParameterT] + + def __init__(self, parameter: ToolParameterT): + self.input_models = [parameter] + + +def parameter_bundle(parameter: ToolParameterT) -> ParameterBundle: + return ParameterBundle(parameter) + + +def parameter_bundle_for_file(filename: str) -> ParameterBundle: + return parameter_bundle(tool_parameter(filename)) + + +def tool_parameter(filename: str) -> ToolParameterT: + return from_input_source(parameter_source(filename)) + + +def parameter_source(filename: str): + tool_source = parameter_tool_source(filename) + input_sources = tool_source.parse_input_pages().page_sources[0].parse_input_sources() + assert len(input_sources) == 1 + return input_sources[0] + + +def parameter_tool_source(basename: str): + path_prefix = os.path.join(galaxy_directory(), "test/functional/tools/parameters", basename) + if os.path.exists(f"{path_prefix}.xml"): + path = f"{path_prefix}.xml" + else: + path = f"{path_prefix}.cwl" + tool_source = get_tool_source(path, macro_paths=[]) + return tool_source diff --git a/lib/tool_shed/managers/tool_state_cache.py b/lib/tool_shed/managers/tool_state_cache.py new file mode 100644 index 000000000000..010ab288a334 --- /dev/null +++ b/lib/tool_shed/managers/tool_state_cache.py @@ -0,0 +1,42 @@ +import json +import os +from typing import ( + Any, + Dict, + Optional, +) + +RAW_CACHED_JSON = Dict[str, Any] + + +class ToolStateCache: + _cache_directory: str + + def __init__(self, cache_directory: str): + if not os.path.exists(cache_directory): + os.makedirs(cache_directory) + self._cache_directory = cache_directory + + def _cache_target(self, tool_id: str, tool_version: str): + # consider breaking this into multiple directories... + cache_target = os.path.join(self._cache_directory, tool_id, tool_version) + return cache_target + + def get_cache_entry_for(self, tool_id: str, tool_version: str) -> Optional[RAW_CACHED_JSON]: + cache_target = self._cache_target(tool_id, tool_version) + if not os.path.exists(cache_target): + return None + with open(cache_target) as f: + return json.load(f) + + def has_cached_entry_for(self, tool_id: str, tool_version: str) -> bool: + cache_target = self._cache_target(tool_id, tool_version) + return os.path.exists(cache_target) + + def insert_cache_entry_for(self, tool_id: str, tool_version: str, entry: RAW_CACHED_JSON) -> None: + cache_target = self._cache_target(tool_id, tool_version) + parent_directory = os.path.dirname(cache_target) + if not os.path.exists(parent_directory): + os.makedirs(parent_directory) + with open(cache_target, "w") as f: + json.dump(entry, f) diff --git a/lib/tool_shed/managers/tools.py b/lib/tool_shed/managers/tools.py index bd648d4903a9..d252fafbadbb 100644 --- a/lib/tool_shed/managers/tools.py +++ b/lib/tool_shed/managers/tools.py @@ -1,8 +1,40 @@ +import os +import tempfile from collections import namedtuple +from typing import ( + List, + Optional, + Tuple, +) from galaxy import exceptions -from tool_shed.context import SessionRequestContext +from galaxy.exceptions import ( + InternalServerError, + ObjectNotFound, +) +from galaxy.tool_shed.metadata.metadata_generator import RepositoryMetadataToolDict +from galaxy.tool_shed.util.basic_util import remove_dir +from galaxy.tool_shed.util.hg_util import ( + clone_repository, + get_changectx_for_changeset, +) +from galaxy.tool_util.parameters import ( + input_models_for_tool_source, + tool_parameter_bundle_from_json, + ToolParameterBundleModel, +) +from galaxy.tool_util.parser import ( + get_tool_source, + ToolSource, +) +from tool_shed.context import ( + ProvidesRepositoriesContext, + SessionRequestContext, +) +from tool_shed.util.common_util import generate_clone_url_for +from tool_shed.webapp.model import RepositoryMetadata from tool_shed.webapp.search.tool_search import ToolSearch +from .trs import trs_tool_id_to_repository_metadata def search(trans: SessionRequestContext, q: str, page: int = 1, page_size: int = 10) -> dict: @@ -42,3 +74,62 @@ def search(trans: SessionRequestContext, q: str, page: int = 1, page_size: int = results = tool_search.search(trans.app, search_term, page, page_size, boosts) results["hostname"] = trans.repositories_hostname return results + + +def get_repository_metadata_tool_dict( + trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str +) -> Tuple[RepositoryMetadata, RepositoryMetadataToolDict]: + name, owner, tool_id = trs_tool_id.split("~", 3) + repository, metadata_by_version = trs_tool_id_to_repository_metadata(trans, trs_tool_id) + if tool_version not in metadata_by_version: + raise ObjectNotFound() + tool_version_repository_metadata: RepositoryMetadata = metadata_by_version[tool_version] + raw_metadata = tool_version_repository_metadata.metadata + tool_dicts: List[RepositoryMetadataToolDict] = raw_metadata.get("tools", []) + for tool_dict in tool_dicts: + if tool_dict["id"] != tool_id or tool_dict["version"] != tool_version: + continue + return tool_version_repository_metadata, tool_dict + raise ObjectNotFound() + + +def tool_input_models_cached_for( + trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None +) -> ToolParameterBundleModel: + tool_state_cache = trans.app.tool_state_cache + raw_json = tool_state_cache.get_cache_entry_for(trs_tool_id, tool_version) + if raw_json is not None: + return tool_parameter_bundle_from_json(raw_json) + bundle = tool_input_models_for(trans, trs_tool_id, tool_version, repository_clone_url=repository_clone_url) + tool_state_cache.insert_cache_entry_for(trs_tool_id, tool_version, bundle.dict()) + return bundle + + +def tool_input_models_for( + trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None +) -> ToolParameterBundleModel: + tool_source = tool_source_for(trans, trs_tool_id, tool_version, repository_clone_url=repository_clone_url) + return input_models_for_tool_source(tool_source) + + +def tool_source_for( + trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None +) -> ToolSource: + rval = get_repository_metadata_tool_dict(trans, trs_tool_id, tool_version) + repository_metadata, tool_version_metadata = rval + tool_config = tool_version_metadata["tool_config"] + + repo = repository_metadata.repository.hg_repo + ctx = get_changectx_for_changeset(repo, repository_metadata.changeset_revision) + work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-tool_source") + if repository_clone_url is None: + repository_clone_url = generate_clone_url_for(trans, repository_metadata.repository) + try: + cloned_ok, error_message = clone_repository(repository_clone_url, work_dir, str(ctx.rev())) + if error_message: + raise InternalServerError("Failed to materialize target repository revision") + path_to_tool = os.path.join(work_dir, tool_config) + tool_source = get_tool_source(path_to_tool) + return tool_source + finally: + remove_dir(work_dir) diff --git a/lib/tool_shed/managers/trs.py b/lib/tool_shed/managers/trs.py index d77fdc7334ee..ebb74220ccd3 100644 --- a/lib/tool_shed/managers/trs.py +++ b/lib/tool_shed/managers/trs.py @@ -74,10 +74,14 @@ def tool_classes() -> List[ToolClass]: return [ToolClass(id="galaxy_tool", name="Galaxy Tool", description="Galaxy XML Tools")] -def trs_tool_id_to_repository(trans: ProvidesRepositoriesContext, trs_tool_id: str) -> Repository: +def trs_tool_id_to_guid(trans: ProvidesRepositoriesContext, trs_tool_id: str) -> str: guid = decode_identifier(trans.repositories_hostname, trs_tool_id) guid = remove_protocol_and_user_from_clone_url(guid) - return guid_to_repository(trans.app, guid) + return guid + + +def trs_tool_id_to_repository(trans: ProvidesRepositoriesContext, trs_tool_id: str) -> Repository: + return guid_to_repository(trans.app, trs_tool_id_to_guid(trans, trs_tool_id)) def get_repository_metadata_by_tool_version( @@ -104,7 +108,7 @@ def get_tools_for(repository_metadata: RepositoryMetadata) -> List[Dict[str, Any def trs_tool_id_to_repository_metadata( trans: ProvidesRepositoriesContext, trs_tool_id: str -) -> Optional[Tuple[Repository, Dict[str, RepositoryMetadata]]]: +) -> Tuple[Repository, Dict[str, RepositoryMetadata]]: tool_guid = decode_identifier(trans.repositories_hostname, trs_tool_id) tool_guid = remove_protocol_and_user_from_clone_url(tool_guid) _, tool_id = tool_guid.rsplit("/", 1) diff --git a/lib/tool_shed/structured_app.py b/lib/tool_shed/structured_app.py index deb0b0ece6be..c3eee0c94299 100644 --- a/lib/tool_shed/structured_app.py +++ b/lib/tool_shed/structured_app.py @@ -3,6 +3,7 @@ from galaxy.structured_app import BasicSharedApp if TYPE_CHECKING: + from tool_shed.managers.tool_state_cache import ToolStateCache from tool_shed.repository_registry import Registry as RepositoryRegistry from tool_shed.repository_types.registry import Registry as RepositoryTypesRegistry from tool_shed.util.hgweb_config import HgWebConfigManager @@ -16,3 +17,4 @@ class ToolShedApp(BasicSharedApp): repository_registry: "RepositoryRegistry" hgweb_config_manager: "HgWebConfigManager" security_agent: "CommunityRBACAgent" + tool_state_cache: "ToolStateCache" diff --git a/lib/tool_shed/test/functional/test_shed_tools.py b/lib/tool_shed/test/functional/test_shed_tools.py index c54dd825e479..5c2a9ea4389e 100644 --- a/lib/tool_shed/test/functional/test_shed_tools.py +++ b/lib/tool_shed/test/functional/test_shed_tools.py @@ -62,9 +62,17 @@ def test_trs_tool_list(self): repository = populator.setup_column_maker_repo(prefix="toolstrsindex") tool_id = populator.tool_guid(self, repository, "Add_a_column1") tool_shed_base, encoded_tool_id = encode_identifier(tool_id) - print(encoded_tool_id) url = f"ga4gh/trs/v2/tools/{encoded_tool_id}" - print(url) tool_response = self.api_interactor.get(url) tool_response.raise_for_status() assert Tool(**tool_response.json()) + + @skip_if_api_v1 + def test_trs_tool_parameter_json_schema(self): + populator = self.populator + repository = populator.setup_column_maker_repo(prefix="toolsparameterschema") + tool_id = populator.tool_guid(self, repository, "Add_a_column1") + tool_shed_base, encoded_tool_id = encode_identifier(tool_id) + url = f"tools/{encoded_tool_id}/versions/1.1.0/parameter_request_schema" + tool_response = self.api_interactor.get(url) + tool_response.raise_for_status() diff --git a/lib/tool_shed/webapp/api2/tools.py b/lib/tool_shed/webapp/api2/tools.py index bd48db71ce01..486a88730909 100644 --- a/lib/tool_shed/webapp/api2/tools.py +++ b/lib/tool_shed/webapp/api2/tools.py @@ -4,10 +4,19 @@ from fastapi import ( Path, Request, + Response, ) +from galaxy.tool_util.parameters import ( + RequestToolState, + to_json_schema_string, + ToolParameterBundleModel, +) from tool_shed.context import SessionRequestContext -from tool_shed.managers.tools import search +from tool_shed.managers.tools import ( + search, + tool_input_models_cached_for, +) from tool_shed.managers.trs import ( get_tool, service_info, @@ -41,6 +50,17 @@ description="See also https://ga4gh.github.io/tool-registry-service-schemas/DataModel/#trs-tool-and-trs-tool-version-ids", ) +TOOL_VERSION_PATH_PARAM: str = Path( + ..., + title="Galaxy Tool Wrapper Version", + description="The full version string defined on the Galaxy tool wrapper.", +) + + +def json_schema_response(pydantic_model) -> Response: + json_str = to_json_schema_string(pydantic_model) + return Response(content=json_str, media_type="application/json") + @router.cbv class FastAPITools: @@ -122,3 +142,32 @@ def trs_get_versions( tool_id: str = TOOL_ID_PATH_PARAM, ) -> List[ToolVersion]: return get_tool(trans, tool_id).versions + + @router.get( + "/api/tools/{tool_id}/versions/{tool_version}/parameter_model", + operation_id="tools__parameter_model", + summary="Return Galaxy's meta model description of the tool's inputs", + ) + def tool_parameters_meta_model( + self, + trans: SessionRequestContext = DependsOnTrans, + tool_id: str = TOOL_ID_PATH_PARAM, + tool_version: str = TOOL_VERSION_PATH_PARAM, + ) -> ToolParameterBundleModel: + return tool_input_models_cached_for(trans, tool_id, tool_version) + + @router.get( + "/api/tools/{tool_id}/versions/{tool_version}/parameter_request_schema", + operation_id="tools__parameter_request_model", + summary="Return a JSON schema description of the tool's inputs for the tool request API that will be added to Galaxy at some point", + description="The tool request schema includes validation of map/reduce concepts that can be consumed by the tool execution API and not just the request for a single execution.", + ) + def tool_state( + self, + trans: SessionRequestContext = DependsOnTrans, + tool_id: str = TOOL_ID_PATH_PARAM, + tool_version: str = TOOL_VERSION_PATH_PARAM, + ) -> Response: + return json_schema_response( + RequestToolState.parameter_model_for(tool_input_models_cached_for(trans, tool_id, tool_version)) + ) diff --git a/lib/tool_shed/webapp/app.py b/lib/tool_shed/webapp/app.py index 58ccf206596c..4083674241a4 100644 --- a/lib/tool_shed/webapp/app.py +++ b/lib/tool_shed/webapp/app.py @@ -33,6 +33,7 @@ from galaxy.structured_app import BasicSharedApp from galaxy.web_stack import application_stack_instance from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager +from tool_shed.managers.tool_state_cache import ToolStateCache from tool_shed.structured_app import ToolShedApp from tool_shed.util.hgweb_config import hgweb_config_manager from tool_shed.webapp.model.migrations import verify_database @@ -83,6 +84,7 @@ def __init__(self, **kwd) -> None: self._register_singleton(SharedModelMapping, model) self._register_singleton(mapping.ToolShedModelMapping, model) self._register_singleton(scoped_session, self.model.context) + self.tool_state_cache = ToolStateCache(self.config.tool_state_cache_dir) self.user_manager = self._register_singleton(UserManager, UserManager(self, app_type="tool_shed")) self.api_keys_manager = self._register_singleton(ApiKeyManager) # initialize the Tool Shed tag handler. diff --git a/lib/tool_shed/webapp/model/__init__.py b/lib/tool_shed/webapp/model/__init__.py index 859b8fe2b096..a31ab4861f4a 100644 --- a/lib/tool_shed/webapp/model/__init__.py +++ b/lib/tool_shed/webapp/model/__init__.py @@ -644,6 +644,8 @@ def __str__(self): class RepositoryMetadata(Dictifiable): + repository: "Repository" + # Once the class has been mapped, all Column items in this table will be available # as instrumented class attributes on RepositoryMetadata. table = Table( diff --git a/test/functional/tools/parameters/cwl_boolean.cwl b/test/functional/tools/parameters/cwl_boolean.cwl new file mode 100644 index 000000000000..be6150d48920 --- /dev/null +++ b/test/functional/tools/parameters/cwl_boolean.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: boolean + +outputs: + output: boolean + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_boolean_optional.cwl b/test/functional/tools/parameters/cwl_boolean_optional.cwl new file mode 100644 index 000000000000..f05516dd5d80 --- /dev/null +++ b/test/functional/tools/parameters/cwl_boolean_optional.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: boolean? + +outputs: + output: boolean? + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_directory.cwl b/test/functional/tools/parameters/cwl_directory.cwl new file mode 100644 index 000000000000..66d3c5353079 --- /dev/null +++ b/test/functional/tools/parameters/cwl_directory.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: Directory + +outputs: + output: Directory + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_file.cwl b/test/functional/tools/parameters/cwl_file.cwl new file mode 100644 index 000000000000..ea48da6d2e82 --- /dev/null +++ b/test/functional/tools/parameters/cwl_file.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: File + +outputs: + output: File + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_float.cwl b/test/functional/tools/parameters/cwl_float.cwl new file mode 100644 index 000000000000..9e7e09469055 --- /dev/null +++ b/test/functional/tools/parameters/cwl_float.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: float + +outputs: + output: float + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_float_optional.cwl b/test/functional/tools/parameters/cwl_float_optional.cwl new file mode 100644 index 000000000000..1bc34fa1bc56 --- /dev/null +++ b/test/functional/tools/parameters/cwl_float_optional.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: float? + +outputs: + output: float? + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_int.cwl b/test/functional/tools/parameters/cwl_int.cwl new file mode 100644 index 000000000000..5ba0a8d5c76c --- /dev/null +++ b/test/functional/tools/parameters/cwl_int.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: int + +outputs: + output: int + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_int_optional.cwl b/test/functional/tools/parameters/cwl_int_optional.cwl new file mode 100644 index 000000000000..63d9c6915862 --- /dev/null +++ b/test/functional/tools/parameters/cwl_int_optional.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: int? + +outputs: + output: int? + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_string.cwl b/test/functional/tools/parameters/cwl_string.cwl new file mode 100644 index 000000000000..571a4cefc6ec --- /dev/null +++ b/test/functional/tools/parameters/cwl_string.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: string + +outputs: + output: string + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/cwl_string_optional.cwl b/test/functional/tools/parameters/cwl_string_optional.cwl new file mode 100644 index 000000000000..fecc8272d99a --- /dev/null +++ b/test/functional/tools/parameters/cwl_string_optional.cwl @@ -0,0 +1,15 @@ +#!/usr/bin/env cwl-runner + +class: ExpressionTool +requirements: + - class: InlineJavascriptRequirement +cwlVersion: v1.2 + +inputs: + parameter: + type: string? + +outputs: + output: string? + +expression: "$({'output': inputs.parameter})" diff --git a/test/functional/tools/parameters/gx_boolean.xml b/test/functional/tools/parameters/gx_boolean.xml new file mode 100644 index 000000000000..e42c9c9b6af6 --- /dev/null +++ b/test/functional/tools/parameters/gx_boolean.xml @@ -0,0 +1,29 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_boolean_optional.xml b/test/functional/tools/parameters/gx_boolean_optional.xml new file mode 100644 index 000000000000..dc667b614a1a --- /dev/null +++ b/test/functional/tools/parameters/gx_boolean_optional.xml @@ -0,0 +1,82 @@ + + > '$output'; +cat '$inputs' >> $inputs_json; + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_color.xml b/test/functional/tools/parameters/gx_color.xml new file mode 100644 index 000000000000..fe158d0e6fb1 --- /dev/null +++ b/test/functional/tools/parameters/gx_color.xml @@ -0,0 +1,21 @@ + + + echo "$parameter" > $out_file1; + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_conditional_boolean.xml b/test/functional/tools/parameters/gx_conditional_boolean.xml new file mode 100644 index 000000000000..7c5feffab0ec --- /dev/null +++ b/test/functional/tools/parameters/gx_conditional_boolean.xml @@ -0,0 +1,101 @@ + + + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_conditional_boolean_checked.xml b/test/functional/tools/parameters/gx_conditional_boolean_checked.xml new file mode 100644 index 000000000000..09fdbd71fe6d --- /dev/null +++ b/test/functional/tools/parameters/gx_conditional_boolean_checked.xml @@ -0,0 +1,53 @@ + + + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_conditional_boolean_discriminate_on_string_value.xml b/test/functional/tools/parameters/gx_conditional_boolean_discriminate_on_string_value.xml new file mode 100644 index 000000000000..6bc790d81ad9 --- /dev/null +++ b/test/functional/tools/parameters/gx_conditional_boolean_discriminate_on_string_value.xml @@ -0,0 +1,113 @@ + + + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_conditional_boolean_optional.xml b/test/functional/tools/parameters/gx_conditional_boolean_optional.xml new file mode 100644 index 000000000000..69fa3d830499 --- /dev/null +++ b/test/functional/tools/parameters/gx_conditional_boolean_optional.xml @@ -0,0 +1,79 @@ + + + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_conditional_conditional_boolean.xml b/test/functional/tools/parameters/gx_conditional_conditional_boolean.xml new file mode 100644 index 000000000000..343f3576cf6e --- /dev/null +++ b/test/functional/tools/parameters/gx_conditional_conditional_boolean.xml @@ -0,0 +1,30 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data.xml b/test/functional/tools/parameters/gx_data.xml new file mode 100644 index 000000000000..ea05c074c033 --- /dev/null +++ b/test/functional/tools/parameters/gx_data.xml @@ -0,0 +1,13 @@ + + > '$output' + ]]> + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_collection.xml b/test/functional/tools/parameters/gx_data_collection.xml new file mode 100644 index 000000000000..5669f2921f64 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_collection.xml @@ -0,0 +1,14 @@ + + > '$output' + ]]> + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_collection_optional.xml b/test/functional/tools/parameters/gx_data_collection_optional.xml new file mode 100644 index 000000000000..9802176c4f76 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_collection_optional.xml @@ -0,0 +1,14 @@ + + > '$output' + ]]> + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_multiple.xml b/test/functional/tools/parameters/gx_data_multiple.xml new file mode 100644 index 000000000000..8529f2c7cac9 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_multiple.xml @@ -0,0 +1,13 @@ + + > '$output' + ]]> + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_multiple_optional.xml b/test/functional/tools/parameters/gx_data_multiple_optional.xml new file mode 100644 index 000000000000..01b63bd83692 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_multiple_optional.xml @@ -0,0 +1,13 @@ + + > '$output' + ]]> + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_data_optional.xml b/test/functional/tools/parameters/gx_data_optional.xml new file mode 100644 index 000000000000..3578d4e436e7 --- /dev/null +++ b/test/functional/tools/parameters/gx_data_optional.xml @@ -0,0 +1,13 @@ + + > '$output' + ]]> + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_float.xml b/test/functional/tools/parameters/gx_float.xml new file mode 100644 index 000000000000..5da8bc9790b9 --- /dev/null +++ b/test/functional/tools/parameters/gx_float.xml @@ -0,0 +1,29 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_float_optional.xml b/test/functional/tools/parameters/gx_float_optional.xml new file mode 100644 index 000000000000..7d2ad2be396e --- /dev/null +++ b/test/functional/tools/parameters/gx_float_optional.xml @@ -0,0 +1,29 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_hidden.xml b/test/functional/tools/parameters/gx_hidden.xml new file mode 100644 index 000000000000..e6da3bfb9279 --- /dev/null +++ b/test/functional/tools/parameters/gx_hidden.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_hidden_optional.xml b/test/functional/tools/parameters/gx_hidden_optional.xml new file mode 100644 index 000000000000..e3969f2a8074 --- /dev/null +++ b/test/functional/tools/parameters/gx_hidden_optional.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_int.xml b/test/functional/tools/parameters/gx_int.xml new file mode 100644 index 000000000000..e6f2e6758d26 --- /dev/null +++ b/test/functional/tools/parameters/gx_int.xml @@ -0,0 +1,29 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_int_optional.xml b/test/functional/tools/parameters/gx_int_optional.xml new file mode 100644 index 000000000000..73b0141c064b --- /dev/null +++ b/test/functional/tools/parameters/gx_int_optional.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_repeat_boolean.xml b/test/functional/tools/parameters/gx_repeat_boolean.xml new file mode 100644 index 000000000000..c57ba06b4838 --- /dev/null +++ b/test/functional/tools/parameters/gx_repeat_boolean.xml @@ -0,0 +1,15 @@ + + > '$output' + ]]> + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_repeat_boolean_min.xml b/test/functional/tools/parameters/gx_repeat_boolean_min.xml new file mode 100644 index 000000000000..7356a6f2ce75 --- /dev/null +++ b/test/functional/tools/parameters/gx_repeat_boolean_min.xml @@ -0,0 +1,15 @@ + + > '$output' + ]]> + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_repeat_data.xml b/test/functional/tools/parameters/gx_repeat_data.xml new file mode 100644 index 000000000000..646a31eb48c4 --- /dev/null +++ b/test/functional/tools/parameters/gx_repeat_data.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_repeat_data_min.xml b/test/functional/tools/parameters/gx_repeat_data_min.xml new file mode 100644 index 000000000000..1d584ad870ac --- /dev/null +++ b/test/functional/tools/parameters/gx_repeat_data_min.xml @@ -0,0 +1,24 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_section_boolean.xml b/test/functional/tools/parameters/gx_section_boolean.xml new file mode 100644 index 000000000000..af948dd3276f --- /dev/null +++ b/test/functional/tools/parameters/gx_section_boolean.xml @@ -0,0 +1,35 @@ + + > '$output' + ]]> + +
+ +
+ + + + + + +
+ +
+ + + + + + + +
+ +
+ + + + + + + + diff --git a/test/functional/tools/parameters/gx_section_data.xml b/test/functional/tools/parameters/gx_section_data.xml new file mode 100644 index 000000000000..76e0e5734c16 --- /dev/null +++ b/test/functional/tools/parameters/gx_section_data.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + +
+ +
+ + + + + + +
+ +
+ + + + diff --git a/test/functional/tools/parameters/gx_select.xml b/test/functional/tools/parameters/gx_select.xml new file mode 100644 index 000000000000..a4f095b7f3cd --- /dev/null +++ b/test/functional/tools/parameters/gx_select.xml @@ -0,0 +1,27 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_select_multiple.xml b/test/functional/tools/parameters/gx_select_multiple.xml new file mode 100644 index 000000000000..0e32bf9653cf --- /dev/null +++ b/test/functional/tools/parameters/gx_select_multiple.xml @@ -0,0 +1,27 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_select_multiple_optional.xml b/test/functional/tools/parameters/gx_select_multiple_optional.xml new file mode 100644 index 000000000000..8e42fb8b14af --- /dev/null +++ b/test/functional/tools/parameters/gx_select_multiple_optional.xml @@ -0,0 +1,27 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_select_optional.xml b/test/functional/tools/parameters/gx_select_optional.xml new file mode 100644 index 000000000000..5f6b63813dd3 --- /dev/null +++ b/test/functional/tools/parameters/gx_select_optional.xml @@ -0,0 +1,27 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_text.xml b/test/functional/tools/parameters/gx_text.xml new file mode 100644 index 000000000000..16707f63e878 --- /dev/null +++ b/test/functional/tools/parameters/gx_text.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/gx_text_optional.xml b/test/functional/tools/parameters/gx_text_optional.xml new file mode 100644 index 000000000000..41fe11cea418 --- /dev/null +++ b/test/functional/tools/parameters/gx_text_optional.xml @@ -0,0 +1,21 @@ + + > '$output' + ]]> + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/parameters/macros.xml b/test/functional/tools/parameters/macros.xml new file mode 100644 index 000000000000..e47d243f75c4 --- /dev/null +++ b/test/functional/tools/parameters/macros.xml @@ -0,0 +1,33 @@ + + + > '$output'; +cat '$inputs' >> '$inputs_json'; + ]]> + + + + + + + + + This is a test tool used to establish and verify the behavior of some aspect of Galaxy's + parameter handling. + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/sample_tool_conf.xml b/test/functional/tools/sample_tool_conf.xml index 45eee3cc16c9..80c032c93601 100644 --- a/test/functional/tools/sample_tool_conf.xml +++ b/test/functional/tools/sample_tool_conf.xml @@ -266,6 +266,8 @@ + +
diff --git a/test/unit/tool_shed/_util.py b/test/unit/tool_shed/_util.py index df50c5270ef2..3002c6a82fab 100644 --- a/test/unit/tool_shed/_util.py +++ b/test/unit/tool_shed/_util.py @@ -18,6 +18,7 @@ from galaxy.util import safe_makedirs from tool_shed.context import ProvidesRepositoriesContext from tool_shed.managers.repositories import upload_tar_and_set_metadata +from tool_shed.managers.tool_state_cache import ToolStateCache from tool_shed.managers.users import create_user from tool_shed.repository_types import util as rt_util from tool_shed.repository_types.registry import Registry as RepositoryTypesRegistry @@ -80,6 +81,7 @@ def __init__(self, temp_directory=None): self.config = TestToolShedConfig(temp_directory) self.security = IdEncodingHelper(id_secret=self.config.id_secret) self.repository_registry = tool_shed.repository_registry.Registry(self) + self.tool_state_cache = ToolStateCache(os.path.join(temp_directory, "tool_state_cache")) @property def security_agent(self): diff --git a/test/unit/tool_shed/test_tool_source.py b/test/unit/tool_shed/test_tool_source.py new file mode 100644 index 000000000000..cb4c4091376a --- /dev/null +++ b/test/unit/tool_shed/test_tool_source.py @@ -0,0 +1,33 @@ +from tool_shed.context import ProvidesRepositoriesContext +from tool_shed.managers.tools import ( + tool_input_models_cached_for, + tool_input_models_for, + tool_source_for, +) +from tool_shed.webapp.model import Repository +from ._util import upload_directories_to_repository + + +def test_get_tool(provides_repositories: ProvidesRepositoriesContext, new_repository: Repository): + upload_directories_to_repository(provides_repositories, new_repository, "column_maker") + owner = new_repository.user.username + name = new_repository.name + encoded_id = f"{owner}~{name}~Add_a_column1" + + repo_path = new_repository.repo_path(app=provides_repositories.app) + tool_source = tool_source_for(provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path) + assert tool_source.parse_id() == "Add_a_column1" + bundle = tool_input_models_for(provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path) + assert len(bundle.input_models) == 3 + + cached_bundle = tool_input_models_cached_for( + provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path + ) + assert len(cached_bundle.input_models) == 3 + + cached_bundle = tool_input_models_cached_for( + provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path + ) + assert len(cached_bundle.input_models) == 3 + + print(RequestToolState.parameter_model_for(cached_bundle).model_json_schema()) diff --git a/test/unit/tool_util/parameter_specification.yml b/test/unit/tool_util/parameter_specification.yml new file mode 100644 index 000000000000..8064a6505e37 --- /dev/null +++ b/test/unit/tool_util/parameter_specification.yml @@ -0,0 +1,681 @@ +# Tools to create. + +# Notes on conditional boolean values... +# - if you set truevalue/falsevalue - it doesn't look like the when can remain +# true/false - so go through and simplify that. means don't need to create test +# cases that test that. Linting also at very least warns on this. + +# - gx_conditional_boolean_empty_default +# - gx_conditional_boolean_empty_else +# - gx_conditional_select_* +# - gx_repeat_select_required +# - gx_repeat_repeat_select_required +# - gx_repeat_conditional_boolean_optional + +# Things to verify: +# - non optional, multi-selects require a selection (see TODO below...) +gx_int: + request_valid: + - parameter: 5 + - parameter: 6 + # galaxy parameters created with a value - so doesn't need to appear in request even though non-optional + - {} + request_invalid: + - parameter: null + - parameter: "null" + - parameter: "None" + - parameter: { 5 } + test_case_valid: + - parameter: 5 + - {} + test_case_invalid: + - parameter: null + - parameter: "5" + +gx_boolean: + request_valid: + - parameter: True + - parameter: False + - {} + request_invalid: + - parameter: {} + - parameter: "true" + # This is borderline... should we allow truevalue/falsevalue in the API. + # Marius and John were on fence here. + - parameter: "mytrue" + - parameter: null + +gx_int_optional: + request_valid: + - parameter: 5 + - parameter: null + - {} + request_invalid: + - parameter: "5" + - parameter: "None" + - parameter: "null" + - parameter: [5] + +gx_text: + request_valid: + - parameter: moocow + - parameter: 'some spaces' + - parameter: '' + request_invalid: + - parameter: 5 + - parameter: null + - parameter: {} + - parameter: { "moo": "cow" } + +gx_text_optional: + request_valid: + - parameter: moocow + - parameter: 'some spaces' + - parameter: '' + - parameter: null + request_invalid: + - parameter: 5 + - parameter: {} + - parameter: { "moo": "cow" } + +gx_select: + request_valid: + - parameter: "--ex1" + - parameter: "ex2" + request_invalid: + # Not allowing selecting booleans by truevalue/falsevalue - don't allow selecting + # selects by label. + - parameter: "Ex1" + # Do not allow lists for non-multi-selects + - parameter: ["ex2"] + - parameter: null + - parameter: {} + - parameter: 5 + - {} + +gx_select_optional: + request_valid: + - parameter: "--ex1" + - parameter: "ex2" + - parameter: null + - {} + request_invalid: + # Not allowing selecting booleans by truevalue/falsevalue - don't allow selecting + # selects by label. + - parameter: "Ex1" + # Do not allow lists for non-multi-selects + - parameter: ["ex2"] + - parameter: {} + - parameter: 5 + +# TODO: confirm null should vaguely not be allowed here +gx_select_multiple: + request_valid: + - parameter: ["--ex1"] + - parameter: ["ex2"] + request_invalid: + - parameter: ["Ex1"] + - parameter: null + - parameter: {} + - parameter: 5 + - {} + +gx_select_multiple_optional: + request_valid: + - parameter: ["--ex1"] + - parameter: ["ex2"] + - {} + - parameter: null + request_invalid: + - parameter: ["Ex1"] + - parameter: {} + - parameter: 5 + +gx_hidden: + request_valid: + - parameter: moocow + - parameter: 'some spaces' + - parameter: '' + request_invalid: + - parameter: null + - parameter: 5 + - parameter: {} + - parameter: { "moo": "cow" } + +gx_hidden_optional: + request_valid: + - parameter: moocow + - parameter: 'some spaces' + - parameter: '' + - parameter: null + request_invalid: + - parameter: 5 + - parameter: {} + - parameter: { "moo": "cow" } + +gx_float: + request_valid: + - parameter: 5 + - parameter: 5.0 + - parameter: 5.0001 + # galaxy parameters created with a value - so doesn't need to appear in request even though non-optional + - {} + request_invalid: + - parameter: null + - parameter: "5" + - parameter: "5.0" + - parameter: { "moo": "cow" } + +gx_float_optional: + request_valid: + - parameter: 5 + - parameter: 5.0 + - parameter: 5.0001 + - parameter: null + - {} + request_invalid: + - parameter: "5" + - parameter: "5.0" + - parameter: {} + - parameter: { "moo": "cow" } + +gx_color: + request_valid: + - parameter: '#aabbcc' + - parameter: '#000000' + request_invalid: + - parameter: null + - parameter: {} + - parameter: '#abcd' + +gx_data: + request_valid: + - parameter: {src: hda, id: abcdabcd} + - parameter: {__class__: "Batch", values: [{src: hdca, id: abcdabcd}]} + request_invalid: + - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} + - parameter: {src: hda, id: 7} + - parameter: {src: hdca, id: abcdabcd} + - parameter: null + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + request_internal_valid: + - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} + - parameter: {src: hda, id: 5} + - parameter: {src: hda, id: 0} + request_internal_invalid: + - parameter: {__class__: "Batch", values: [{src: hdca, id: abcdabcd}]} + - parameter: {src: hda, id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + job_internal_valid: + - parameter: {src: hda, id: 7} + job_internal_invalid: + # valid request but after the job has been created, map/reduce concepts should have been + # expanded out. + - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} + - parameter: {src: hda, id: abcdabcd} + + +gx_data_optional: + request_valid: + - parameter: {src: hda, id: abcdabcd} + - parameter: null + - {} + request_invalid: + - parameter: {src: hda, id: 5} + - parameter: {src: hdca, id: abcdabcd} + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - parameter: true + - parameter: 5 + - parameter: "5" + request_internal_valid: + - parameter: {src: hda, id: 5} + - parameter: null + - {} + request_internal_invalid: + - parameter: {src: hda, id: abcdabcd} + - parameter: {src: hdca, id: abcdabcd} + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - parameter: true + - parameter: 5 + - parameter: "5" + +gx_data_multiple: + request_valid: + - parameter: {src: hda, id: abcdabcd} + - parameter: {src: hdca, id: abcdabcd} + - parameter: [{src: hda, id: abcdabcd}] + - parameter: [{src: hdca, id: abcdabcd}] + - parameter: [{src: hdca, id: abcdabcd}, {src: hda, id: abcdabcd}] + request_invalid: + - parameter: {src: hda, id: 5} + - parameter: [{src: hdca, id: 5}, {src: hda, id: 5}] + - parameter: null + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + request_internal_valid: + - parameter: {src: hda, id: 5} + - parameter: {src: hdca, id: 5} + - parameter: [{src: hda, id: 5}] + - parameter: [{src: hdca, id: 5}] + - parameter: [{src: hdca, id: 5}, {src: hda, id: 5}] + request_internal_invalid: + - parameter: {src: hda, id: abcdabcd} + - parameter: [{src: hdca, id: abcdabcd}, {src: hda, id: abcdabcd}] + - parameter: null + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + +gx_data_multiple_optional: + request_valid: + - parameter: {src: hda, id: abcdabcd} + - parameter: {src: hdca, id: abcdabcd} + - parameter: [{src: hda, id: abcdabcd}] + - parameter: [{src: hdca, id: abcdabcd}] + - parameter: [{src: hdca, id: abcdabcd}, {src: hda, id: abcdabcd}] + - parameter: null + - {} + request_invalid: + - parameter: {src: hda, id: 5} + - parameter: {} + - parameter: true + - parameter: 5 + - parameter: "5" + request_internal_valid: + - parameter: {src: hda, id: 5} + - parameter: {src: hdca, id: 5} + - parameter: [{src: hda, id: 5}] + - parameter: [{src: hdca, id: 5}] + - parameter: [{src: hdca, id: 5}, {src: hda, id: 5}] + - parameter: null + - {} + request_internal_invalid: + - parameter: {src: hda, id: abcdabcd} + - parameter: {} + - parameter: true + - parameter: 5 + - parameter: "5" + +gx_data_collection: + request_valid: + - parameter: {src: hdca, id: abcdabcd} + request_invalid: + - parameter: {src: hdca, id: 7} + - parameter: null + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + request_internal_valid: + - parameter: {src: hdca, id: 5} + request_internal_invalid: + - parameter: {src: hdca, id: abcdabcd} + - parameter: null + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + +gx_data_collection_optional: + request_valid: + - parameter: {src: hdca, id: abcdabcd} + - parameter: null + - {} + request_invalid: + - parameter: {src: hdca, id: 7} + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: true + - parameter: 5 + - parameter: "5" + - parameter: {} + request_internal_valid: + - parameter: {src: hdca, id: 5} + - parameter: null + - {} + request_internal_invalid: + - parameter: {src: hdca, id: abcdabcd} + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: true + - parameter: 5 + - parameter: "5" + - parameter: {} + +gx_conditional_boolean: + request_valid: + - conditional_parameter: + test_parameter: true + integer_parameter: 1 + - conditional_parameter: + test_parameter: true + integer_parameter: 2 + - conditional_parameter: + test_parameter: false + boolean_parameter: true + # Test parameter has default and so does it "case" - so this should be fine + - {} + # The boolean_parameter is optional so just setting a test_parameter is fine + - conditional_parameter: + test_parameter: true + - conditional_parameter: + test_parameter: false + # if test parameter is missing, it should be false in this case (TODO: test inverse) + # so boolean_parameter or either type or missing should be fine. + - conditional_parameter: + boolean_parameter: true + - conditional_parameter: + boolean_parameter: false + - conditional_parameter: {} + request_invalid: + - conditional_parameter: + test_parameter: false + integer_parameter: 1 + - conditional_parameter: + test_parameter: null + - conditional_parameter: + test_parameter: true + integer_parameter: "1" + - conditional_parameter: + test_parameter: true + integer_parameter: null + # if test parameter is missing, it should be false in this case + # in that case having an integer_parameter is not acceptable. + - conditional_parameter: + integer_parameter: 5 + +gx_conditional_boolean_checked: + request_valid: + # if no test parameter is defined, the default is 'checked' so the test + # parameter is true. + - conditional_parameter: + integer_parameter: 5 + - conditional_parameter: + integer_parameter: 0 + + request_invalid: + # if test parameter is missing, it should be true (it is 'checked') in this case + # in that case having a boolean_parameter is not acceptable. + - conditional_parameter: + boolean_parameter: false + +gx_conditional_conditional_boolean: + request_valid: + - outer_conditional_parameter: + outer_test_parameter: false + boolean_parameter: true + - outer_conditional_parameter: + outer_test_parameter: true + inner_conditional_parameter: + inner_test_parameter: true + integer_parameter: 5 + - outer_conditional_parameter: + outer_test_parameter: true + inner_conditional_parameter: + inner_test_parameter: false + boolean_parameter: true + # Test parameter has default and so does it "case" - so this should be fine + - {} + request_invalid: + - outer_conditional_parameter: + outer_test_parameter: true + boolean_parameter: true + - outer_conditional_parameter: + outer_test_parameter: true + inner_conditional_parameter: + inner_test_parameter: false + integer_parameter: 5 + - outer_conditional_parameter: + outer_test_parameter: true + inner_conditional_parameter: + inner_test_parameter: true + integer_parameter: true + +gx_repeat_boolean: + request_valid: + - parameter: + - { boolean_parameter: true } + - parameter: [] + - parameter: + - { boolean_parameter: true } + - { boolean_parameter: false } + - parameter: [{}] + - parameter: [{}, {}] + request_invalid: + - parameter: + - { boolean_parameter: 4 } + - parameter: + - { foo: 4 } + - parameter: + - { boolean_parameter: true } + - { boolean_parameter: false } + - { boolean_parameter: 4 } + - parameter: 5 + +gx_repeat_boolean_min: + request_valid: + - parameter: + - { boolean_parameter: true } + - { boolean_parameter: false } + - parameter: [{}, {}] + request_invalid: + - parameter: [] + - parameter: [{}] + - parameter: + - { boolean_parameter: true } + - parameter: + - { boolean_parameter: 4 } + - parameter: + - { foo: 4 } + - parameter: + - { boolean_parameter: true } + - { boolean_parameter: false } + - { boolean_parameter: 4 } + - parameter: 5 + +gx_repeat_data: + request_valid: + - parameter: + - { data_parameter: {src: hda, id: abcdabcd} } + - { data_parameter: {src: hda, id: abcdabcd} } + - parameter: [] + - parameter: + - { data_parameter: {src: hda, id: abcdabcd} } + request_invalid: + - parameter: [{}, {}] + - parameter: [{}] + - parameter: 5 + request_internal_valid: + - parameter: + - { data_parameter: { src: hda, id: 5 } } + request_internal_invalid: + - parameter: + - { data_parameter: { src: hda, id: abcdabcd } } + +gx_repeat_data_min: + request_valid: + - parameter: + - { data_parameter: {src: hda, id: abcdabcd} } + - { data_parameter: {src: hda, id: abcdabcd} } + request_invalid: + - parameter: [] + - parameter: + - { data_parameter: {src: hda, id: abcdabcd} } + - parameter: [{}, {}] + - parameter: [{}] + - parameter: 5 + request_internal_valid: + - parameter: + - { data_parameter: { src: hda, id: 5 } } + - { data_parameter: { src: hda, id: 5 } } + request_internal_invalid: + - parameter: + - { data_parameter: { src: hda, id: abcdabcd } } + - parameter: + - { data_parameter: { src: hda, id: 5 } } + +gx_section_boolean: + request_valid: + - parameter: { boolean_parameter: true } + # booleans are optional in requests, so this should be fine? + - {} + request_invalid: + - parameter: { boolean_parameter: 4 } + +gx_section_data: + request_valid: + - parameter: { data_parameter: { src: hda, id: abcdabcd } } + request_invalid: + - parameter: { data_parameter: 4 } + - parameter: { data_parameter: { src: hda, id: 5 } } + # data parameter is non-optional, so this should be invalid (unlike boolean parameter above) + # - {} + request_internal_valid: + - parameter: { data_parameter: { src: hda, id: 5 } } + request_internal_invalid: + - parameter: { data_parameter: { src: hda, id: abcdabcd } } + +cwl_int: + request_valid: + - parameter: 5 + request_invalid: + - parameter: "5" + - {} + - parameter: null + - parameter: "None" + + +# TODO: Not a thing perhaps? +# cwl_null: +# request_valid: +# - parameter: null +# - {} +# request_invalid: +# - parameter: "5" +# - parameter: 5 +# - parameter: {} + +cwl_int_optional: + request_valid: + - parameter: 5 + - parameter: null + request_invalid: + - parameter: "5" + - {} + - parameter: "None" + +cwl_float: + request_valid: + - parameter: 5 + - parameter: 5.0 + request_invalid: + - parameter: null + - parameter: "5" + - {} + - parameter: "None" + +cwl_float_optional: + request_valid: + - parameter: 5 + - parameter: 5.0 + - parameter: null + request_invalid: + - parameter: "5" + - {} + - parameter: "None" + +cwl_string: + request_valid: + - parameter: "moo" + - parameter: "" + request_invalid: + - parameter: null + - {} + - parameter: 5 + +cwl_string_optional: + request_valid: + - parameter: "moo" + - parameter: "" + - parameter: null + request_invalid: + - {} + - parameter: 5 + +cwl_boolean: + request_valid: + - parameter: true + - parameter: false + request_invalid: + - parameter: null + - {} + - parameter: 5 + - parameter: "true" + - parameter: "True" + +cwl_boolean_optional: + request_valid: + - parameter: true + - parameter: false + - parameter: null + request_invalid: + - {} + - parameter: 5 + - parameter: "true" + - parameter: "True" + +cwl_file: + request_valid: + - parameter: {src: hda, id: abcdabcd} + request_invalid: + - parameter: {src: hda, id: 7} + - parameter: {src: hdca, id: abcdabcd} + - parameter: null + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + +cwl_directory: + request_valid: + - parameter: {src: hda, id: abcdabcd} + request_invalid: + - parameter: {src: hda, id: 7} + - parameter: {src: hdca, id: abcdabcd} + - parameter: null + - parameter: {src: fooda, id: abcdabcd} + - parameter: {id: abcdabcd} + - parameter: {} + - {} + - parameter: true + - parameter: 5 + - parameter: "5" + diff --git a/test/unit/tool_util/test_parameter_specification.py b/test/unit/tool_util/test_parameter_specification.py new file mode 100644 index 000000000000..1ad8332230fc --- /dev/null +++ b/test/unit/tool_util/test_parameter_specification.py @@ -0,0 +1,226 @@ +from functools import partial +from typing import ( + Any, + Callable, + Dict, + List, +) + +import yaml + +from galaxy.exceptions import RequestParameterInvalidException +from galaxy.tool_util.parameters import ( + decode, + encode, + RequestInternalToolState, + RequestToolState, + ToolParameterModel, + validate_internal_job, + validate_internal_request, + validate_request, + validate_test_case, +) +from galaxy.tool_util.parameters.json import to_json_schema_string +from galaxy.tool_util.unittest_utils.parameters import ( + parameter_bundle, + parameter_bundle_for_file, + tool_parameter, +) +from galaxy.util.resources import resource_string + + +def specification_object(): + try: + yaml_str = resource_string(__package__, "parameter_specification.yml") + except AttributeError: + # hack for the main() function below where this file is interpreted as part of the + # Galaxy tree. + yaml_str = open("test/unit/tool_util/parameter_specification.yml").read() + return yaml.safe_load(yaml_str) + + +def test_specification(): + parameter_spec = specification_object() + for file in parameter_spec.keys(): + _test_file(file, parameter_spec) + + +def test_single(): + # _test_file("gx_int") + # _test_file("gx_float") + # _test_file("gx_boolean") + # _test_file("gx_int_optional") + # _test_file("gx_float_optional") + # _test_file("gx_conditional_boolean") + # _test_file("gx_conditional_conditional_boolean") + _test_file("gx_conditional_boolean_checked") + + +def _test_file(file: str, specification=None): + spec = specification or specification_object() + combos = spec[file] + tool_parameter_model = tool_parameter(file) + for valid_or_invalid, tests in combos.items(): + if valid_or_invalid == "request_valid": + _assert_requests_validate(tool_parameter_model, tests) + elif valid_or_invalid == "request_invalid": + _assert_requests_invalid(tool_parameter_model, tests) + elif valid_or_invalid == "request_internal_valid": + _assert_internal_requests_validate(tool_parameter_model, tests) + elif valid_or_invalid == "request_internal_invalid": + _assert_internal_requests_invalid(tool_parameter_model, tests) + elif valid_or_invalid == "job_internal_valid": + _assert_internal_jobs_validate(tool_parameter_model, tests) + elif valid_or_invalid == "job_internal_invalid": + _assert_internal_jobs_invalid(tool_parameter_model, tests) + elif valid_or_invalid == "test_case_valid": + _assert_test_cases_validate(tool_parameter_model, tests) + elif valid_or_invalid == "test_case_invalid": + _assert_test_cases_invalid(tool_parameter_model, tests) + + # Assume request validation will work here. + if "request_internal_valid" not in combos and "request_valid" in combos: + _assert_internal_requests_validate(tool_parameter_model, combos["request_valid"]) + if "request_internal_invalid" not in combos and "request_invalid" in combos: + _assert_internal_requests_invalid(tool_parameter_model, combos["request_invalid"]) + + +def _for_each(test: Callable, parameter: ToolParameterModel, requests: List[Dict[str, Any]]) -> None: + for request in requests: + test(parameter, request) + + +def _assert_request_validates(parameter, request) -> None: + try: + validate_request(parameter_bundle(parameter), request) + except RequestParameterInvalidException as e: + raise AssertionError(f"Parameter {parameter} failed to validate request {request}. {e}") + + +def _assert_request_invalid(parameter, request) -> None: + exc = None + try: + validate_request(parameter_bundle(parameter), request) + except RequestParameterInvalidException as e: + exc = e + assert exc is not None, f"Parameter {parameter} didn't result in validation error on request {request} as expected." + + +def _assert_internal_request_validates(parameter, request) -> None: + try: + validate_internal_request(parameter_bundle(parameter), request) + except RequestParameterInvalidException as e: + raise AssertionError(f"Parameter {parameter} failed to validate internal request {request}. {e}") + + +def _assert_internal_request_invalid(parameter, request) -> None: + exc = None + try: + validate_internal_request(parameter_bundle(parameter), request) + except RequestParameterInvalidException as e: + exc = e + assert ( + exc is not None + ), f"Parameter {parameter} didn't result in validation error on internal request {request} as expected." + + +def _assert_internal_job_validates(parameter, request) -> None: + try: + validate_internal_job(parameter_bundle(parameter), request) + except RequestParameterInvalidException as e: + raise AssertionError(f"Parameter {parameter} failed to validate internal job description {request}. {e}") + + +def _assert_internal_job_invalid(parameter, request) -> None: + exc = None + try: + validate_internal_job(parameter_bundle(parameter), request) + except RequestParameterInvalidException as e: + exc = e + assert ( + exc is not None + ), f"Parameter {parameter} didn't result in validation error on internal job description {request} as expected." + + +def _assert_test_case_validates(parameter, test_case) -> None: + try: + validate_test_case(parameter_bundle(parameter), test_case) + except RequestParameterInvalidException as e: + raise AssertionError(f"Parameter {parameter} failed to validate test_case {test_case}. {e}") + + +def _assert_test_case_invalid(parameter, test_case) -> None: + exc = None + try: + validate_test_case(parameter_bundle(parameter), test_case) + except RequestParameterInvalidException as e: + exc = e + assert ( + exc is not None + ), f"Parameter {parameter} didn't result in validation error on test_case {test_case} as expected." + + +_assert_requests_validate = partial(_for_each, _assert_request_validates) +_assert_requests_invalid = partial(_for_each, _assert_request_invalid) +_assert_internal_requests_validate = partial(_for_each, _assert_internal_request_validates) +_assert_internal_requests_invalid = partial(_for_each, _assert_internal_request_invalid) +_assert_internal_jobs_validate = partial(_for_each, _assert_internal_job_validates) +_assert_internal_jobs_invalid = partial(_for_each, _assert_internal_job_invalid) +_assert_test_cases_validate = partial(_for_each, _assert_test_case_validates) +_assert_test_cases_invalid = partial(_for_each, _assert_test_case_invalid) + + +def decode_val(val: str) -> int: + assert val == "abcdabcd" + return 5 + + +def test_decode_gx_data(): + input_bundle = parameter_bundle_for_file("gx_data") + + request_tool_state = RequestToolState({"parameter": {"src": "hda", "id": "abcdabcd"}}) + request_internal_tool_state = decode(request_tool_state, input_bundle, decode_val) + assert request_internal_tool_state.input_state["parameter"]["id"] == 5 + assert request_internal_tool_state.input_state["parameter"]["src"] == "hda" + + +def test_decode_gx_int(): + input_bundle = parameter_bundle_for_file("gx_int") + + request_tool_state = RequestToolState({"parameter": 5}) + request_internal_tool_state = decode(request_tool_state, input_bundle, decode_val) + assert request_internal_tool_state.input_state["parameter"] == 5 + + +def test_json_schema_for_conditional(): + input_bundle = parameter_bundle_for_file("gx_conditional_boolean") + tool_state = RequestToolState.parameter_model_for(input_bundle) + print(to_json_schema_string(tool_state)) + + +def test_encode_gx_data(): + input_bundle = parameter_bundle_for_file("gx_data") + + def encode_val(val: int) -> str: + assert val == 5 + return "abcdabcd" + + request_internal_tool_state = RequestInternalToolState({"parameter": {"src": "hda", "id": 5}}) + request_tool_state = encode(request_internal_tool_state, input_bundle, encode_val) + assert request_tool_state.input_state["parameter"]["id"] == "abcdabcd" + assert request_tool_state.input_state["parameter"]["src"] == "hda" + + +if __name__ == "__main__": + parameter_spec = specification_object() + parameter_models_json = {} + for file in parameter_spec.keys(): + tool_parameter_model = tool_parameter(file) + parameter_models_json[file] = tool_parameter_model.dict() + yaml_str = yaml.safe_dump(parameter_models_json) + with open("client/src/components/Tool/parameter_models.yml", "w") as f: + f.write("# auto generated file for JavaScript testing, do not modify manually\n") + f.write("# -----\n") + f.write('# PYTHONPATH="lib" python test/unit/tool_util/test_parameter_specification.py\n') + f.write("# -----\n") + f.write(yaml_str) From aca192eeafd802628c45e292314583ebb468de42 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 29 Nov 2022 17:17:38 -0500 Subject: [PATCH 08/12] SVG generation for tool state docs... --- doc/source/dev/tool_state_api.plantuml.svg | 46 ++++++ .../dev/tool_state_state_classes.plantuml.svg | 153 ++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 doc/source/dev/tool_state_api.plantuml.svg create mode 100644 doc/source/dev/tool_state_state_classes.plantuml.svg diff --git a/doc/source/dev/tool_state_api.plantuml.svg b/doc/source/dev/tool_state_api.plantuml.svg new file mode 100644 index 000000000000..3aaf86c1ee00 --- /dev/null +++ b/doc/source/dev/tool_state_api.plantuml.svg @@ -0,0 +1,46 @@ +API RequestAPI RequestJobs APIJobs APIJob ServiceJob ServiceDatabaseDatabaseTaskQueueTaskQueueHTTP JSONTo boundaryBuild RequestToolStateValidate RequestToolState (pydantic)decode() RequestToolStateinto RequestInternalToolStateSerialize RequestInternalToolStateQueue QueueJobs with reference topersisted RequestInternalToolStateJobCreateResponse(pydantic model)JobCreateResponse(as json) \ No newline at end of file diff --git a/doc/source/dev/tool_state_state_classes.plantuml.svg b/doc/source/dev/tool_state_state_classes.plantuml.svg new file mode 100644 index 000000000000..b0c086bf18b0 --- /dev/null +++ b/doc/source/dev/tool_state_state_classes.plantuml.svg @@ -0,0 +1,153 @@ +galaxy.tool_util.parameters.stateToolStatestate_representation: strinput_state: Dict[str, Any]validate(input_models: ToolParameterBundle)_to_base_model(input_models: ToolParameterBundle): Optional[Type[BaseModel]]RequestToolStatestate_representation = "request"_to_base_model(input_models: ToolParameterBundle): Type[BaseModel]Object references of the form{src: "hda", id: <encoded_id>}.Allow mapping/reduce constructs.RequestInternalToolStatestate_representation = "request_internal"_to_base_model(input_models: ToolParameterBundle): Type[BaseModel]Object references of the form{src: "hda", id: <decoded_id>}.Allow mapping/reduce constructs.JobInternalToolStatestate_representation = "job_internal"_to_base_model(input_models: ToolParameterBundle): Type[BaseModel]Object references of the form{src: "hda", id: <decoded_id>}.Mapping constructs expanded out.(Defaults are inserted?)decodeexpand \ No newline at end of file From 6dec2f482e2e20fafc269986fde1fcae75595cee Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 30 Jun 2024 11:16:26 -0400 Subject: [PATCH 09/12] Stock tools. --- lib/galaxy/tools/stock.py | 33 ++++++++++++++++++++++ lib/galaxy/util/__init__.py | 12 ++++++-- lib/tool_shed/managers/tools.py | 37 +++++++++++++++++++++++++ test/unit/app/tools/test_stock.py | 16 +++++++++++ test/unit/tool_shed/test_tool_source.py | 7 ++++- 5 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 lib/galaxy/tools/stock.py create mode 100644 test/unit/app/tools/test_stock.py diff --git a/lib/galaxy/tools/stock.py b/lib/galaxy/tools/stock.py new file mode 100644 index 000000000000..1a86354f1370 --- /dev/null +++ b/lib/galaxy/tools/stock.py @@ -0,0 +1,33 @@ +"""Reason about stock tools based on ToolSource abstractions.""" + +from pathlib import Path + +from lxml.etree import XMLSyntaxError + +# Set GALAXY_INCLUDES_ROOT from tool shed to point this at a Galaxy root +# (once we are running the tool shed from packages not rooted with Galaxy). +import galaxy.tools +from galaxy.tool_util.parser import get_tool_source +from galaxy.util import galaxy_directory +from galaxy.util.resources import files + + +def stock_tool_paths(): + yield from _walk_directory_for_tools(files(galaxy.tools)) + yield from _walk_directory_for_tools(Path(galaxy_directory()) / "test" / "functional" / "tools") + + +def stock_tool_sources(): + for stock_tool_path in stock_tool_paths(): + try: + yield get_tool_source(str(stock_tool_path)) + except XMLSyntaxError: + continue + + +def _walk_directory_for_tools(path): + if path.is_file() and path.name.endswith(".xml"): + yield path + elif path.is_dir(): + for directory in path.iterdir(): + yield from _walk_directory_for_tools(directory) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index a0ba3205225f..607207eb872c 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1738,12 +1738,20 @@ def safe_str_cmp(a, b): return rv == 0 +# never load packages this way (won't work for installed packages), +# but while we're working on packaging everything this can be a way to point +# an installed Galaxy at a Galaxy root for things like tools. Ultimately +# this all needs to be packaged, but we have some very old PRs working on this +# that are pretty tricky and shouldn't slow current development. +GALAXY_INCLUDES_ROOT = os.environ.get("GALAXY_INCLUDES_ROOT") + + # Don't use this directly, prefer method version that "works" with packaged Galaxy. -galaxy_root_path = Path(__file__).parent.parent.parent.parent +galaxy_root_path = Path(GALAXY_INCLUDES_ROOT) if GALAXY_INCLUDES_ROOT else Path(__file__).parent.parent.parent.parent def galaxy_directory() -> str: - if in_packages(): + if in_packages() and not GALAXY_INCLUDES_ROOT: # This will work only when running pytest from /packages// cwd = Path.cwd() path = cwd.parent.parent diff --git a/lib/tool_shed/managers/tools.py b/lib/tool_shed/managers/tools.py index d252fafbadbb..a881c13f046d 100644 --- a/lib/tool_shed/managers/tools.py +++ b/lib/tool_shed/managers/tools.py @@ -2,6 +2,7 @@ import tempfile from collections import namedtuple from typing import ( + Dict, List, Optional, Tuple, @@ -27,6 +28,7 @@ get_tool_source, ToolSource, ) +from galaxy.tools.stock import stock_tool_sources from tool_shed.context import ( ProvidesRepositoriesContext, SessionRequestContext, @@ -36,6 +38,8 @@ from tool_shed.webapp.search.tool_search import ToolSearch from .trs import trs_tool_id_to_repository_metadata +STOCK_TOOL_SOURCES: Optional[Dict[str, Dict[str, ToolSource]]] = None + def search(trans: SessionRequestContext, q: str, page: int = 1, page_size: int = 10) -> dict: """ @@ -114,6 +118,18 @@ def tool_input_models_for( def tool_source_for( trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None +) -> ToolSource: + if "~" in trs_tool_id: + return _shed_tool_source_for(trans, trs_tool_id, tool_version, repository_clone_url) + else: + tool_source = _stock_tool_source_for(trs_tool_id, tool_version) + if tool_source is None: + raise ObjectNotFound() + return tool_source + + +def _shed_tool_source_for( + trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None ) -> ToolSource: rval = get_repository_metadata_tool_dict(trans, trs_tool_id, tool_version) repository_metadata, tool_version_metadata = rval @@ -133,3 +149,24 @@ def tool_source_for( return tool_source finally: remove_dir(work_dir) + + +def _stock_tool_source_for(tool_id: str, tool_version: str) -> Optional[ToolSource]: + _init_stock_tool_sources() + assert STOCK_TOOL_SOURCES + tool_version_sources = STOCK_TOOL_SOURCES.get(tool_id) + if tool_version_sources is None: + return None + return tool_version_sources.get(tool_version) + + +def _init_stock_tool_sources() -> None: + global STOCK_TOOL_SOURCES + if STOCK_TOOL_SOURCES is None: + STOCK_TOOL_SOURCES = {} + for tool_source in stock_tool_sources(): + tool_id = tool_source.parse_id() + tool_version = tool_source.parse_version() + if tool_id not in STOCK_TOOL_SOURCES: + STOCK_TOOL_SOURCES[tool_id] = {} + STOCK_TOOL_SOURCES[tool_id][tool_version] = tool_source diff --git a/test/unit/app/tools/test_stock.py b/test/unit/app/tools/test_stock.py new file mode 100644 index 000000000000..25aeca875319 --- /dev/null +++ b/test/unit/app/tools/test_stock.py @@ -0,0 +1,16 @@ +from galaxy.tools.stock import ( + stock_tool_paths, + stock_tool_sources, +) + + +def test_stock_tool_paths(): + file_names = [f.name for f in list(stock_tool_paths())] + assert "merge_collection.xml" in file_names + assert "meme.xml" in file_names + assert "output_auto_format.xml" in file_names + + +def test_stock_tool_sources(): + tool_source = next(stock_tool_sources()) + assert tool_source.parse_id() diff --git a/test/unit/tool_shed/test_tool_source.py b/test/unit/tool_shed/test_tool_source.py index cb4c4091376a..4925cd52cd3f 100644 --- a/test/unit/tool_shed/test_tool_source.py +++ b/test/unit/tool_shed/test_tool_source.py @@ -30,4 +30,9 @@ def test_get_tool(provides_repositories: ProvidesRepositoriesContext, new_reposi ) assert len(cached_bundle.input_models) == 3 - print(RequestToolState.parameter_model_for(cached_bundle).model_json_schema()) + +def test_stock_bundle(provides_repositories: ProvidesRepositoriesContext): + cached_bundle = tool_input_models_cached_for( + provides_repositories, "__ZIP_COLLECTION__", "1.0.0", repository_clone_url=None + ) + assert len(cached_bundle.input_models) == 2 From bcb25383c8a06914461821216f20327fb5c6792e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 5 Jul 2024 12:30:25 -0400 Subject: [PATCH 10/12] Improved tool API. --- .../schemas/tool_shed_config_schema.yml | 6 +- lib/tool_shed/managers/model_cache.py | 64 +++++++++++++++ lib/tool_shed/managers/tool_state_cache.py | 42 ---------- lib/tool_shed/managers/tools.py | 80 +++++++++++++++---- lib/tool_shed/structured_app.py | 4 +- lib/tool_shed/webapp/api2/tools.py | 17 ++-- lib/tool_shed/webapp/app.py | 4 +- test/unit/tool_shed/_util.py | 4 +- test/unit/tool_shed/test_model_cache.py | 53 ++++++++++++ test/unit/tool_shed/test_tool_source.py | 20 ++--- 10 files changed, 210 insertions(+), 84 deletions(-) create mode 100644 lib/tool_shed/managers/model_cache.py delete mode 100644 lib/tool_shed/managers/tool_state_cache.py create mode 100644 test/unit/tool_shed/test_model_cache.py diff --git a/lib/galaxy/config/schemas/tool_shed_config_schema.yml b/lib/galaxy/config/schemas/tool_shed_config_schema.yml index 2a1eee2b533f..b7b6bfee049e 100644 --- a/lib/galaxy/config/schemas/tool_shed_config_schema.yml +++ b/lib/galaxy/config/schemas/tool_shed_config_schema.yml @@ -102,12 +102,12 @@ mapping: the repositories and tools within the Tool Shed given that you specify the following two config options. - tool_state_cache_dir: + model_cache_dir: type: str - default: database/tool_state_cache + default: database/model_cache required: false desc: | - Cache directory for tool state. + Cache directory for Pydantic model objects. repo_name_boost: type: float diff --git a/lib/tool_shed/managers/model_cache.py b/lib/tool_shed/managers/model_cache.py new file mode 100644 index 000000000000..9ce8c205e901 --- /dev/null +++ b/lib/tool_shed/managers/model_cache.py @@ -0,0 +1,64 @@ +import json +import os +from typing import ( + Any, + Dict, + Optional, + Type, + TypeVar, +) + +from pydantic import BaseModel + +from galaxy.util.hash_util import md5_hash_str + +RAW_CACHED_JSON = Dict[str, Any] + + +def hash_model(model_class: Type[BaseModel]) -> str: + return md5_hash_str(json.dumps(model_class.model_json_schema())) + + +MODEL_HASHES: Dict[Type[BaseModel], str] = {} + + +M = TypeVar("M", bound=BaseModel) + + +def ensure_model_has_hash(model_class: Type[BaseModel]) -> None: + if model_class not in MODEL_HASHES: + MODEL_HASHES[model_class] = hash_model(model_class) + + +class ModelCache: + _cache_directory: str + + def __init__(self, cache_directory: str): + if not os.path.exists(cache_directory): + os.makedirs(cache_directory) + self._cache_directory = cache_directory + + def _cache_target(self, model_class: Type[M], tool_id: str, tool_version: str) -> str: + ensure_model_has_hash(model_class) + # consider breaking this into multiple directories... + cache_target = os.path.join(self._cache_directory, MODEL_HASHES[model_class], tool_id, tool_version) + return cache_target + + def get_cache_entry_for(self, model_class: Type[M], tool_id: str, tool_version: str) -> Optional[M]: + cache_target = self._cache_target(model_class, tool_id, tool_version) + if not os.path.exists(cache_target): + return None + with open(cache_target) as f: + return model_class.model_validate(json.load(f)) + + def has_cached_entry_for(self, model_class: Type[M], tool_id: str, tool_version: str) -> bool: + cache_target = self._cache_target(model_class, tool_id, tool_version) + return os.path.exists(cache_target) + + def insert_cache_entry_for(self, model_object: M, tool_id: str, tool_version: str) -> None: + cache_target = self._cache_target(model_object.__class__, tool_id, tool_version) + parent_directory = os.path.dirname(cache_target) + if not os.path.exists(parent_directory): + os.makedirs(parent_directory) + with open(cache_target, "w") as f: + json.dump(model_object.dict(), f) diff --git a/lib/tool_shed/managers/tool_state_cache.py b/lib/tool_shed/managers/tool_state_cache.py deleted file mode 100644 index 010ab288a334..000000000000 --- a/lib/tool_shed/managers/tool_state_cache.py +++ /dev/null @@ -1,42 +0,0 @@ -import json -import os -from typing import ( - Any, - Dict, - Optional, -) - -RAW_CACHED_JSON = Dict[str, Any] - - -class ToolStateCache: - _cache_directory: str - - def __init__(self, cache_directory: str): - if not os.path.exists(cache_directory): - os.makedirs(cache_directory) - self._cache_directory = cache_directory - - def _cache_target(self, tool_id: str, tool_version: str): - # consider breaking this into multiple directories... - cache_target = os.path.join(self._cache_directory, tool_id, tool_version) - return cache_target - - def get_cache_entry_for(self, tool_id: str, tool_version: str) -> Optional[RAW_CACHED_JSON]: - cache_target = self._cache_target(tool_id, tool_version) - if not os.path.exists(cache_target): - return None - with open(cache_target) as f: - return json.load(f) - - def has_cached_entry_for(self, tool_id: str, tool_version: str) -> bool: - cache_target = self._cache_target(tool_id, tool_version) - return os.path.exists(cache_target) - - def insert_cache_entry_for(self, tool_id: str, tool_version: str, entry: RAW_CACHED_JSON) -> None: - cache_target = self._cache_target(tool_id, tool_version) - parent_directory = os.path.dirname(cache_target) - if not os.path.exists(parent_directory): - os.makedirs(parent_directory) - with open(cache_target, "w") as f: - json.dump(entry, f) diff --git a/lib/tool_shed/managers/tools.py b/lib/tool_shed/managers/tools.py index a881c13f046d..c2c97e5c7fbd 100644 --- a/lib/tool_shed/managers/tools.py +++ b/lib/tool_shed/managers/tools.py @@ -8,6 +8,8 @@ Tuple, ) +from pydantic import BaseModel + from galaxy import exceptions from galaxy.exceptions import ( InternalServerError, @@ -21,13 +23,16 @@ ) from galaxy.tool_util.parameters import ( input_models_for_tool_source, - tool_parameter_bundle_from_json, - ToolParameterBundleModel, + ToolParameterT, ) from galaxy.tool_util.parser import ( get_tool_source, ToolSource, ) +from galaxy.tool_util.parser.interface import ( + Citation, + XrefDict, +) from galaxy.tools.stock import stock_tool_sources from tool_shed.context import ( ProvidesRepositoriesContext, @@ -41,6 +46,53 @@ STOCK_TOOL_SOURCES: Optional[Dict[str, Dict[str, ToolSource]]] = None +# parse the tool source with galaxy.util abstractions to provide a bit richer +# information about the tool than older tool shed abstractions. +class ParsedTool(BaseModel): + id: str + version: Optional[str] + name: str + description: Optional[str] + inputs: List[ToolParameterT] + citations: List[Citation] + license: Optional[str] + profile: Optional[str] + edam_operations: List[str] + edam_topics: List[str] + xrefs: List[XrefDict] + help: Optional[str] + + +def _parse_tool(tool_source: ToolSource) -> ParsedTool: + id = tool_source.parse_id() + version = tool_source.parse_version() + name = tool_source.parse_name() + description = tool_source.parse_description() + inputs = input_models_for_tool_source(tool_source).input_models + citations = tool_source.parse_citations() + license = tool_source.parse_license() + profile = tool_source.parse_profile() + edam_operations = tool_source.parse_edam_operations() + edam_topics = tool_source.parse_edam_topics() + xrefs = tool_source.parse_xrefs() + help = tool_source.parse_help() + + return ParsedTool( + id=id, + version=version, + name=name, + description=description, + profile=profile, + inputs=inputs, + license=license, + citations=citations, + edam_operations=edam_operations, + edam_topics=edam_topics, + xrefs=xrefs, + help=help, + ) + + def search(trans: SessionRequestContext, q: str, page: int = 1, page_size: int = 10) -> dict: """ Perform the search over TS tools index. @@ -97,23 +149,23 @@ def get_repository_metadata_tool_dict( raise ObjectNotFound() -def tool_input_models_cached_for( +def parsed_tool_model_cached_for( trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None -) -> ToolParameterBundleModel: - tool_state_cache = trans.app.tool_state_cache - raw_json = tool_state_cache.get_cache_entry_for(trs_tool_id, tool_version) - if raw_json is not None: - return tool_parameter_bundle_from_json(raw_json) - bundle = tool_input_models_for(trans, trs_tool_id, tool_version, repository_clone_url=repository_clone_url) - tool_state_cache.insert_cache_entry_for(trs_tool_id, tool_version, bundle.dict()) - return bundle +) -> ParsedTool: + model_cache = trans.app.model_cache + parsed_tool = model_cache.get_cache_entry_for(ParsedTool, trs_tool_id, tool_version) + if parsed_tool is not None: + return parsed_tool + parsed_tool = parsed_tool_model_for(trans, trs_tool_id, tool_version, repository_clone_url=repository_clone_url) + model_cache.insert_cache_entry_for(parsed_tool, trs_tool_id, tool_version) + return parsed_tool -def tool_input_models_for( +def parsed_tool_model_for( trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None -) -> ToolParameterBundleModel: +) -> ParsedTool: tool_source = tool_source_for(trans, trs_tool_id, tool_version, repository_clone_url=repository_clone_url) - return input_models_for_tool_source(tool_source) + return _parse_tool(tool_source) def tool_source_for( diff --git a/lib/tool_shed/structured_app.py b/lib/tool_shed/structured_app.py index c3eee0c94299..8fd828f9f5e1 100644 --- a/lib/tool_shed/structured_app.py +++ b/lib/tool_shed/structured_app.py @@ -3,7 +3,7 @@ from galaxy.structured_app import BasicSharedApp if TYPE_CHECKING: - from tool_shed.managers.tool_state_cache import ToolStateCache + from tool_shed.managers.model_cache import ModelCache from tool_shed.repository_registry import Registry as RepositoryRegistry from tool_shed.repository_types.registry import Registry as RepositoryTypesRegistry from tool_shed.util.hgweb_config import HgWebConfigManager @@ -17,4 +17,4 @@ class ToolShedApp(BasicSharedApp): repository_registry: "RepositoryRegistry" hgweb_config_manager: "HgWebConfigManager" security_agent: "CommunityRBACAgent" - tool_state_cache: "ToolStateCache" + model_cache: "ModelCache" diff --git a/lib/tool_shed/webapp/api2/tools.py b/lib/tool_shed/webapp/api2/tools.py index 486a88730909..7f549e80d1c0 100644 --- a/lib/tool_shed/webapp/api2/tools.py +++ b/lib/tool_shed/webapp/api2/tools.py @@ -10,12 +10,12 @@ from galaxy.tool_util.parameters import ( RequestToolState, to_json_schema_string, - ToolParameterBundleModel, ) from tool_shed.context import SessionRequestContext from tool_shed.managers.tools import ( + parsed_tool_model_cached_for, + ParsedTool, search, - tool_input_models_cached_for, ) from tool_shed.managers.trs import ( get_tool, @@ -144,17 +144,17 @@ def trs_get_versions( return get_tool(trans, tool_id).versions @router.get( - "/api/tools/{tool_id}/versions/{tool_version}/parameter_model", + "/api/tools/{tool_id}/versions/{tool_version}", operation_id="tools__parameter_model", summary="Return Galaxy's meta model description of the tool's inputs", ) - def tool_parameters_meta_model( + def show_tool( self, trans: SessionRequestContext = DependsOnTrans, tool_id: str = TOOL_ID_PATH_PARAM, tool_version: str = TOOL_VERSION_PATH_PARAM, - ) -> ToolParameterBundleModel: - return tool_input_models_cached_for(trans, tool_id, tool_version) + ) -> ParsedTool: + return parsed_tool_model_cached_for(trans, tool_id, tool_version) @router.get( "/api/tools/{tool_id}/versions/{tool_version}/parameter_request_schema", @@ -168,6 +168,5 @@ def tool_state( tool_id: str = TOOL_ID_PATH_PARAM, tool_version: str = TOOL_VERSION_PATH_PARAM, ) -> Response: - return json_schema_response( - RequestToolState.parameter_model_for(tool_input_models_cached_for(trans, tool_id, tool_version)) - ) + parsed_tool = parsed_tool_model_cached_for(trans, tool_id, tool_version) + return json_schema_response(RequestToolState.parameter_model_for(parsed_tool.inputs)) diff --git a/lib/tool_shed/webapp/app.py b/lib/tool_shed/webapp/app.py index 4083674241a4..e046301497ad 100644 --- a/lib/tool_shed/webapp/app.py +++ b/lib/tool_shed/webapp/app.py @@ -33,7 +33,7 @@ from galaxy.structured_app import BasicSharedApp from galaxy.web_stack import application_stack_instance from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager -from tool_shed.managers.tool_state_cache import ToolStateCache +from tool_shed.managers.model_cache import ModelCache from tool_shed.structured_app import ToolShedApp from tool_shed.util.hgweb_config import hgweb_config_manager from tool_shed.webapp.model.migrations import verify_database @@ -84,7 +84,7 @@ def __init__(self, **kwd) -> None: self._register_singleton(SharedModelMapping, model) self._register_singleton(mapping.ToolShedModelMapping, model) self._register_singleton(scoped_session, self.model.context) - self.tool_state_cache = ToolStateCache(self.config.tool_state_cache_dir) + self.model_cache = ModelCache(self.config.model_cache_dir) self.user_manager = self._register_singleton(UserManager, UserManager(self, app_type="tool_shed")) self.api_keys_manager = self._register_singleton(ApiKeyManager) # initialize the Tool Shed tag handler. diff --git a/test/unit/tool_shed/_util.py b/test/unit/tool_shed/_util.py index 3002c6a82fab..e20bb17c3c13 100644 --- a/test/unit/tool_shed/_util.py +++ b/test/unit/tool_shed/_util.py @@ -17,8 +17,8 @@ from galaxy.security.idencoding import IdEncodingHelper from galaxy.util import safe_makedirs from tool_shed.context import ProvidesRepositoriesContext +from tool_shed.managers.model_cache import ModelCache from tool_shed.managers.repositories import upload_tar_and_set_metadata -from tool_shed.managers.tool_state_cache import ToolStateCache from tool_shed.managers.users import create_user from tool_shed.repository_types import util as rt_util from tool_shed.repository_types.registry import Registry as RepositoryTypesRegistry @@ -81,7 +81,7 @@ def __init__(self, temp_directory=None): self.config = TestToolShedConfig(temp_directory) self.security = IdEncodingHelper(id_secret=self.config.id_secret) self.repository_registry = tool_shed.repository_registry.Registry(self) - self.tool_state_cache = ToolStateCache(os.path.join(temp_directory, "tool_state_cache")) + self.model_cache = ModelCache(os.path.join(temp_directory, "model_cache")) @property def security_agent(self): diff --git a/test/unit/tool_shed/test_model_cache.py b/test/unit/tool_shed/test_model_cache.py new file mode 100644 index 000000000000..89ebb8d54042 --- /dev/null +++ b/test/unit/tool_shed/test_model_cache.py @@ -0,0 +1,53 @@ +from pydantic import ( + BaseModel, + ConfigDict, +) + +from tool_shed.managers.model_cache import ( + hash_model, + ModelCache, +) + + +class Moo(BaseModel): + foo: int + + +class MooLike(BaseModel): + model_config = ConfigDict(title="Moo") + foo: int + + +class NewMoo(BaseModel): + model_config = ConfigDict(title="Moo") + foo: int + new_prop: str + + +def test_hash(): + hash_moo_1 = hash_model(Moo) + hash_moo_2 = hash_model(Moo) + assert hash_moo_1 == hash_moo_2 + + +def test_hash_by_value(): + hash_moo_1 = hash_model(Moo) + hash_moo_like = hash_model(MooLike) + assert hash_moo_1 == hash_moo_like + + +def test_hash_different_on_updates(): + hash_moo_1 = hash_model(Moo) + hash_moo_new = hash_model(NewMoo) + assert hash_moo_1 != hash_moo_new + + +def cache_dict(tmp_path): + model_cache = ModelCache(tmp_path) + assert not model_cache.has_cached_entry_for(Moo, "moo", "1.0") + assert None is model_cache.get_cache_entry_for(Moo, "moo", "1.0") + model_cache.insert_cache_entry_for(Moo(foo=4), "moo", "1.0") + moo = model_cache.get_cache_entry_for(Moo, "moo", "1.0") + assert moo + assert moo.foo == 4 + assert model_cache.has_cached_entry_for(Moo, "moo", "1.0") diff --git a/test/unit/tool_shed/test_tool_source.py b/test/unit/tool_shed/test_tool_source.py index 4925cd52cd3f..601d4d63df54 100644 --- a/test/unit/tool_shed/test_tool_source.py +++ b/test/unit/tool_shed/test_tool_source.py @@ -1,7 +1,7 @@ from tool_shed.context import ProvidesRepositoriesContext from tool_shed.managers.tools import ( - tool_input_models_cached_for, - tool_input_models_for, + parsed_tool_model_cached_for, + parsed_tool_model_for, tool_source_for, ) from tool_shed.webapp.model import Repository @@ -17,22 +17,22 @@ def test_get_tool(provides_repositories: ProvidesRepositoriesContext, new_reposi repo_path = new_repository.repo_path(app=provides_repositories.app) tool_source = tool_source_for(provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path) assert tool_source.parse_id() == "Add_a_column1" - bundle = tool_input_models_for(provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path) - assert len(bundle.input_models) == 3 + bundle = parsed_tool_model_for(provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path) + assert len(bundle.inputs) == 3 - cached_bundle = tool_input_models_cached_for( + cached_bundle = parsed_tool_model_cached_for( provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path ) - assert len(cached_bundle.input_models) == 3 + assert len(cached_bundle.inputs) == 3 - cached_bundle = tool_input_models_cached_for( + cached_bundle = parsed_tool_model_cached_for( provides_repositories, encoded_id, "1.2.0", repository_clone_url=repo_path ) - assert len(cached_bundle.input_models) == 3 + assert len(cached_bundle.inputs) == 3 def test_stock_bundle(provides_repositories: ProvidesRepositoriesContext): - cached_bundle = tool_input_models_cached_for( + cached_bundle = parsed_tool_model_cached_for( provides_repositories, "__ZIP_COLLECTION__", "1.0.0", repository_clone_url=None ) - assert len(cached_bundle.input_models) == 2 + assert len(cached_bundle.inputs) == 2 From ae6464e61035590ccc5e36204515a5ada66f0fdf Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 10 Jul 2024 14:20:20 -0400 Subject: [PATCH 11/12] Rework ParsedTool for reuse outside of the tool shed. --- lib/galaxy/tool_util/models.py | 74 ++++++++++++++++++++++++++++++ lib/tool_shed/managers/tools.py | 61 ++---------------------- lib/tool_shed/webapp/api2/tools.py | 2 +- 3 files changed, 79 insertions(+), 58 deletions(-) create mode 100644 lib/galaxy/tool_util/models.py diff --git a/lib/galaxy/tool_util/models.py b/lib/galaxy/tool_util/models.py new file mode 100644 index 000000000000..4ee1c8c70b9c --- /dev/null +++ b/lib/galaxy/tool_util/models.py @@ -0,0 +1,74 @@ +"""Define the ParsedTool model representing metadata extracted from a tool's source. + +This is abstraction exported by newer tool shed APIS (circa 2024) and should be sufficient +for reasoning about tool state externally from Galaxy. +""" + +from typing import ( + List, + Optional, +) + +from pydantic import BaseModel + +from .parameters import ( + input_models_for_tool_source, + ToolParameterT, +) +from .parser.interface import ( + Citation, + ToolSource, + XrefDict, +) +from .parser.output_models import ( + from_tool_source, + ToolOutput, +) + + +class ParsedTool(BaseModel): + id: str + version: Optional[str] + name: str + description: Optional[str] + inputs: List[ToolParameterT] + outputs: List[ToolOutput] + citations: List[Citation] + license: Optional[str] + profile: Optional[str] + edam_operations: List[str] + edam_topics: List[str] + xrefs: List[XrefDict] + help: Optional[str] + + +def parse_tool(tool_source: ToolSource) -> ParsedTool: + id = tool_source.parse_id() + version = tool_source.parse_version() + name = tool_source.parse_name() + description = tool_source.parse_description() + inputs = input_models_for_tool_source(tool_source).input_models + outputs = from_tool_source(tool_source) + citations = tool_source.parse_citations() + license = tool_source.parse_license() + profile = tool_source.parse_profile() + edam_operations = tool_source.parse_edam_operations() + edam_topics = tool_source.parse_edam_topics() + xrefs = tool_source.parse_xrefs() + help = tool_source.parse_help() + + return ParsedTool( + id=id, + version=version, + name=name, + description=description, + profile=profile, + inputs=inputs, + outputs=outputs, + license=license, + citations=citations, + edam_operations=edam_operations, + edam_topics=edam_topics, + xrefs=xrefs, + help=help, + ) diff --git a/lib/tool_shed/managers/tools.py b/lib/tool_shed/managers/tools.py index c2c97e5c7fbd..1075ac414305 100644 --- a/lib/tool_shed/managers/tools.py +++ b/lib/tool_shed/managers/tools.py @@ -8,8 +8,6 @@ Tuple, ) -from pydantic import BaseModel - from galaxy import exceptions from galaxy.exceptions import ( InternalServerError, @@ -21,18 +19,14 @@ clone_repository, get_changectx_for_changeset, ) -from galaxy.tool_util.parameters import ( - input_models_for_tool_source, - ToolParameterT, +from galaxy.tool_util.models import ( + parse_tool, + ParsedTool, ) from galaxy.tool_util.parser import ( get_tool_source, ToolSource, ) -from galaxy.tool_util.parser.interface import ( - Citation, - XrefDict, -) from galaxy.tools.stock import stock_tool_sources from tool_shed.context import ( ProvidesRepositoriesContext, @@ -46,53 +40,6 @@ STOCK_TOOL_SOURCES: Optional[Dict[str, Dict[str, ToolSource]]] = None -# parse the tool source with galaxy.util abstractions to provide a bit richer -# information about the tool than older tool shed abstractions. -class ParsedTool(BaseModel): - id: str - version: Optional[str] - name: str - description: Optional[str] - inputs: List[ToolParameterT] - citations: List[Citation] - license: Optional[str] - profile: Optional[str] - edam_operations: List[str] - edam_topics: List[str] - xrefs: List[XrefDict] - help: Optional[str] - - -def _parse_tool(tool_source: ToolSource) -> ParsedTool: - id = tool_source.parse_id() - version = tool_source.parse_version() - name = tool_source.parse_name() - description = tool_source.parse_description() - inputs = input_models_for_tool_source(tool_source).input_models - citations = tool_source.parse_citations() - license = tool_source.parse_license() - profile = tool_source.parse_profile() - edam_operations = tool_source.parse_edam_operations() - edam_topics = tool_source.parse_edam_topics() - xrefs = tool_source.parse_xrefs() - help = tool_source.parse_help() - - return ParsedTool( - id=id, - version=version, - name=name, - description=description, - profile=profile, - inputs=inputs, - license=license, - citations=citations, - edam_operations=edam_operations, - edam_topics=edam_topics, - xrefs=xrefs, - help=help, - ) - - def search(trans: SessionRequestContext, q: str, page: int = 1, page_size: int = 10) -> dict: """ Perform the search over TS tools index. @@ -165,7 +112,7 @@ def parsed_tool_model_for( trans: ProvidesRepositoriesContext, trs_tool_id: str, tool_version: str, repository_clone_url: Optional[str] = None ) -> ParsedTool: tool_source = tool_source_for(trans, trs_tool_id, tool_version, repository_clone_url=repository_clone_url) - return _parse_tool(tool_source) + return parse_tool(tool_source) def tool_source_for( diff --git a/lib/tool_shed/webapp/api2/tools.py b/lib/tool_shed/webapp/api2/tools.py index 7f549e80d1c0..be5d04da9aeb 100644 --- a/lib/tool_shed/webapp/api2/tools.py +++ b/lib/tool_shed/webapp/api2/tools.py @@ -7,6 +7,7 @@ Response, ) +from galaxy.tool_util.models import ParsedTool from galaxy.tool_util.parameters import ( RequestToolState, to_json_schema_string, @@ -14,7 +15,6 @@ from tool_shed.context import SessionRequestContext from tool_shed.managers.tools import ( parsed_tool_model_cached_for, - ParsedTool, search, ) from tool_shed.managers.trs import ( From fc57099fc663aee77c1b1fc3866b7c1bb92abb9c Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 15 Jul 2024 09:47:42 -0400 Subject: [PATCH 12/12] Rebuild tool shed 2.0 API schema... --- .../webapp/frontend/src/schema/schema.ts | 1099 +++++++++++++++++ 1 file changed, 1099 insertions(+) diff --git a/lib/tool_shed/webapp/frontend/src/schema/schema.ts b/lib/tool_shed/webapp/frontend/src/schema/schema.ts index af27cb9318fc..6e53a87611b5 100644 --- a/lib/tool_shed/webapp/frontend/src/schema/schema.ts +++ b/lib/tool_shed/webapp/frontend/src/schema/schema.ts @@ -168,6 +168,17 @@ export interface paths { */ put: operations["tools__build_search_index"] } + "/api/tools/{tool_id}/versions/{tool_version}": { + /** Return Galaxy's meta model description of the tool's inputs */ + get: operations["tools__parameter_model"] + } + "/api/tools/{tool_id}/versions/{tool_version}/parameter_request_schema": { + /** + * Return a JSON schema description of the tool's inputs for the tool request API that will be added to Galaxy at some point + * @description The tool request schema includes validation of map/reduce concepts that can be consumed by the tool execution API and not just the request for a single execution. + */ + get: operations["tools__parameter_request_model"] + } "/api/users": { /** * Index @@ -259,6 +270,48 @@ export interface components { /** Files */ files?: string[] | null } + /** BooleanParameterModel */ + BooleanParameterModel: { + /** Argument */ + argument?: string | null + /** Falsevalue */ + falsevalue?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_boolean + * @constant + * @enum {string} + */ + parameter_type?: "gx_boolean" + /** Truevalue */ + truevalue?: string | null + /** + * Value + * @default false + */ + value?: boolean | null + } /** BuildSearchIndexResponse */ BuildSearchIndexResponse: { /** Repositories Indexed */ @@ -295,6 +348,119 @@ export interface components { */ type: string } + /** Citation */ + Citation: { + /** Content */ + content: string + /** Type */ + type: string + } + /** ColorParameterModel */ + ColorParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_color + * @constant + * @enum {string} + */ + parameter_type?: "gx_color" + /** Value */ + value?: string | null + } + /** ConditionalParameterModel */ + ConditionalParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_conditional + * @constant + * @enum {string} + */ + parameter_type?: "gx_conditional" + /** Test Parameter */ + test_parameter: + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["SelectParameterModel"] + /** Whens */ + whens: components["schemas"]["ConditionalWhen"][] + } + /** ConditionalWhen */ + ConditionalWhen: { + /** Discriminator */ + discriminator: boolean | string + /** Is Default When */ + is_default_when: boolean + /** Parameters */ + parameters: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + | components["schemas"]["TextParameterModel"] + | components["schemas"]["IntegerParameterModel"] + | components["schemas"]["FloatParameterModel"] + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["HiddenParameterModel"] + | components["schemas"]["SelectParameterModel"] + | components["schemas"]["DataParameterModel"] + | components["schemas"]["DataCollectionParameterModel"] + | components["schemas"]["DirectoryUriParameterModel"] + | components["schemas"]["RulesParameterModel"] + | components["schemas"]["ColorParameterModel"] + | components["schemas"]["ConditionalParameterModel"] + | components["schemas"]["RepeatParameterModel"] + | components["schemas"]["SectionParameterModel"] + )[] + } /** CreateCategoryRequest */ CreateCategoryRequest: { /** Description */ @@ -332,6 +498,246 @@ export interface components { /** Username */ username: string } + /** CwlBooleanParameterModel */ + CwlBooleanParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_boolean + * @constant + * @enum {string} + */ + parameter_type?: "cwl_boolean" + } + /** CwlDirectoryParameterModel */ + CwlDirectoryParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default cwl_directory + * @constant + * @enum {string} + */ + parameter_type?: "cwl_directory" + } + /** CwlFileParameterModel */ + CwlFileParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default cwl_file + * @constant + * @enum {string} + */ + parameter_type?: "cwl_file" + } + /** CwlFloatParameterModel */ + CwlFloatParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_float + * @constant + * @enum {string} + */ + parameter_type?: "cwl_float" + } + /** CwlIntegerParameterModel */ + CwlIntegerParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_integer + * @constant + * @enum {string} + */ + parameter_type?: "cwl_integer" + } + /** CwlNullParameterModel */ + CwlNullParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_null + * @constant + * @enum {string} + */ + parameter_type?: "cwl_null" + } + /** CwlStringParameterModel */ + CwlStringParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_string + * @constant + * @enum {string} + */ + parameter_type?: "cwl_string" + } + /** CwlUnionParameterModel */ + CwlUnionParameterModel: { + /** Name */ + name: string + /** + * Parameter Type + * @default cwl_union + * @constant + * @enum {string} + */ + parameter_type?: "cwl_union" + /** Parameters */ + parameters: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + )[] + } + /** DataCollectionParameterModel */ + DataCollectionParameterModel: { + /** Argument */ + argument?: string | null + /** Collection Type */ + collection_type?: string | null + /** + * Extensions + * @default [ + * "data" + * ] + */ + extensions?: string[] + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_data_collection + * @constant + * @enum {string} + */ + parameter_type?: "gx_data_collection" + } + /** DataParameterModel */ + DataParameterModel: { + /** Argument */ + argument?: string | null + /** + * Extensions + * @default [ + * "data" + * ] + */ + extensions?: string[] + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Max */ + max?: number | null + /** Min */ + min?: number | null + /** + * Multiple + * @default false + */ + multiple?: boolean + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_data + * @constant + * @enum {string} + */ + parameter_type?: "gx_data" + } /** * DescriptorType * @enum {string} @@ -375,11 +781,150 @@ export interface components { /** User Id */ user_id: string } + /** DirectoryUriParameterModel */ + DirectoryUriParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @constant + * @enum {string} + */ + parameter_type: "gx_directory_uri" + /** Value */ + value: string | null + } /** FailedRepositoryUpdateMessage */ FailedRepositoryUpdateMessage: { /** Err Msg */ err_msg: string } + /** FilePatternDatasetCollectionDescription */ + FilePatternDatasetCollectionDescription: { + /** Assign Primary Output */ + assign_primary_output: boolean + /** Directory */ + directory: string | null + /** + * Discover Via + * @constant + * @enum {string} + */ + discover_via: "pattern" + /** Format */ + format: string | null + /** Match Relative Path */ + match_relative_path: boolean + /** Pattern */ + pattern: string + /** Recurse */ + recurse: boolean + /** + * Sort Comp + * @enum {string} + */ + sort_comp: "lexical" | "numeric" + /** + * Sort Key + * @enum {string} + */ + sort_key: "filename" | "name" | "designation" | "dbkey" + /** Visible */ + visible: boolean + } + /** FloatParameterModel */ + FloatParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Max */ + max?: number | null + /** Min */ + min?: number | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_float + * @constant + * @enum {string} + */ + parameter_type?: "gx_float" + /** Value */ + value?: number | null + } + /** HiddenParameterModel */ + HiddenParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_hidden + * @constant + * @enum {string} + */ + parameter_type?: "gx_hidden" + } /** ImageData */ ImageData: { /** @@ -419,6 +964,51 @@ export interface components { metadata_info?: components["schemas"]["RepositoryMetadataInstallInfo"] | null repo_info?: components["schemas"]["RepositoryExtraInstallInfo"] | null } + /** IntegerParameterModel */ + IntegerParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Max */ + max?: number | null + /** Min */ + min?: number | null + /** Name */ + name: string + /** Optional */ + optional: boolean + /** + * Parameter Type + * @default gx_integer + * @constant + * @enum {string} + */ + parameter_type?: "gx_integer" + /** Value */ + value?: number | null + } + /** LabelValue */ + LabelValue: { + /** Label */ + label: string + /** Selected */ + selected: boolean + /** Value */ + value: string + } /** MessageExceptionModel */ MessageExceptionModel: { /** Err Code */ @@ -440,6 +1030,127 @@ export interface components { */ url: string } + /** ParsedTool */ + ParsedTool: { + /** Citations */ + citations: components["schemas"]["Citation"][] + /** Description */ + description: string | null + /** Edam Operations */ + edam_operations: string[] + /** Edam Topics */ + edam_topics: string[] + /** Help */ + help: string | null + /** Id */ + id: string + /** Inputs */ + inputs: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + | components["schemas"]["TextParameterModel"] + | components["schemas"]["IntegerParameterModel"] + | components["schemas"]["FloatParameterModel"] + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["HiddenParameterModel"] + | components["schemas"]["SelectParameterModel"] + | components["schemas"]["DataParameterModel"] + | components["schemas"]["DataCollectionParameterModel"] + | components["schemas"]["DirectoryUriParameterModel"] + | components["schemas"]["RulesParameterModel"] + | components["schemas"]["ColorParameterModel"] + | components["schemas"]["ConditionalParameterModel"] + | components["schemas"]["RepeatParameterModel"] + | components["schemas"]["SectionParameterModel"] + )[] + /** License */ + license: string | null + /** Name */ + name: string + /** Outputs */ + outputs: ( + | components["schemas"]["ToolOutputDataset"] + | components["schemas"]["ToolOutputCollection"] + | components["schemas"]["ToolOutputText"] + | components["schemas"]["ToolOutputInteger"] + | components["schemas"]["ToolOutputFloat"] + | components["schemas"]["ToolOutputBoolean"] + )[] + /** Profile */ + profile: string | null + /** Version */ + version: string | null + /** Xrefs */ + xrefs: components["schemas"]["XrefDict"][] + } + /** RepeatParameterModel */ + RepeatParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Max */ + max?: number | null + /** Min */ + min?: number | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_repeat + * @constant + * @enum {string} + */ + parameter_type?: "gx_repeat" + /** Parameters */ + parameters: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + | components["schemas"]["TextParameterModel"] + | components["schemas"]["IntegerParameterModel"] + | components["schemas"]["FloatParameterModel"] + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["HiddenParameterModel"] + | components["schemas"]["SelectParameterModel"] + | components["schemas"]["DataParameterModel"] + | components["schemas"]["DataCollectionParameterModel"] + | components["schemas"]["DirectoryUriParameterModel"] + | components["schemas"]["RulesParameterModel"] + | components["schemas"]["ColorParameterModel"] + | components["schemas"]["ConditionalParameterModel"] + | components["schemas"]["RepeatParameterModel"] + | components["schemas"]["SectionParameterModel"] + )[] + } /** RepositoriesByCategory */ RepositoriesByCategory: { /** Description */ @@ -695,6 +1406,134 @@ export interface components { /** Stop Time */ stop_time: string } + /** RulesParameterModel */ + RulesParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_rules + * @constant + * @enum {string} + */ + parameter_type?: "gx_rules" + } + /** SectionParameterModel */ + SectionParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_section + * @constant + * @enum {string} + */ + parameter_type?: "gx_section" + /** Parameters */ + parameters: ( + | components["schemas"]["CwlIntegerParameterModel"] + | components["schemas"]["CwlFloatParameterModel"] + | components["schemas"]["CwlStringParameterModel"] + | components["schemas"]["CwlBooleanParameterModel"] + | components["schemas"]["CwlNullParameterModel"] + | components["schemas"]["CwlFileParameterModel"] + | components["schemas"]["CwlDirectoryParameterModel"] + | components["schemas"]["CwlUnionParameterModel"] + | components["schemas"]["TextParameterModel"] + | components["schemas"]["IntegerParameterModel"] + | components["schemas"]["FloatParameterModel"] + | components["schemas"]["BooleanParameterModel"] + | components["schemas"]["HiddenParameterModel"] + | components["schemas"]["SelectParameterModel"] + | components["schemas"]["DataParameterModel"] + | components["schemas"]["DataCollectionParameterModel"] + | components["schemas"]["DirectoryUriParameterModel"] + | components["schemas"]["RulesParameterModel"] + | components["schemas"]["ColorParameterModel"] + | components["schemas"]["ConditionalParameterModel"] + | components["schemas"]["RepeatParameterModel"] + | components["schemas"]["SectionParameterModel"] + )[] + } + /** SelectParameterModel */ + SelectParameterModel: { + /** Argument */ + argument?: string | null + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Multiple */ + multiple: boolean + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** Options */ + options?: components["schemas"]["LabelValue"][] | null + /** + * Parameter Type + * @default gx_select + * @constant + * @enum {string} + */ + parameter_type?: "gx_select" + } /** Service */ Service: { /** @@ -764,6 +1603,51 @@ export interface components { */ version: string } + /** TextParameterModel */ + TextParameterModel: { + /** + * Area + * @default false + */ + area?: boolean + /** Argument */ + argument?: string | null + /** + * Default Options + * @default [] + */ + default_options?: components["schemas"]["LabelValue"][] + /** Help */ + help?: string | null + /** + * Hidden + * @default false + */ + hidden?: boolean + /** + * Is Dynamic + * @default false + */ + is_dynamic?: boolean + /** Label */ + label?: string | null + /** Name */ + name: string + /** + * Optional + * @default false + */ + optional?: boolean + /** + * Parameter Type + * @default gx_text + * @constant + * @enum {string} + */ + parameter_type?: "gx_text" + /** Value */ + value?: string | null + } /** Tool */ Tool: { /** @@ -839,6 +1723,149 @@ export interface components { */ name?: string | null } + /** ToolOutputBoolean */ + ToolOutputBoolean: { + /** Hidden */ + hidden: boolean + /** Label */ + label: string | null + /** Name */ + name: string + /** + * Type + * @constant + * @enum {string} + */ + type: "boolean" + } + /** ToolOutputCollection */ + ToolOutputCollection: { + /** Hidden */ + hidden: boolean + /** Label */ + label: string | null + /** Name */ + name: string + structure: components["schemas"]["ToolOutputCollectionStructure"] + /** + * Type + * @constant + * @enum {string} + */ + type: "collection" + } + /** ToolOutputCollectionStructure */ + ToolOutputCollectionStructure: { + /** Collection Type */ + collection_type: string | null + /** Collection Type From Rules */ + collection_type_from_rules: string | null + /** Collection Type Source */ + collection_type_source: string | null + /** Discover Datasets */ + discover_datasets: + | ( + | components["schemas"]["FilePatternDatasetCollectionDescription"] + | components["schemas"]["ToolProvidedMetadataDatasetCollection"] + )[] + | null + /** Structured Like */ + structured_like: string | null + } + /** ToolOutputDataset */ + ToolOutputDataset: { + /** Discover Datasets */ + discover_datasets: + | ( + | components["schemas"]["FilePatternDatasetCollectionDescription"] + | components["schemas"]["ToolProvidedMetadataDatasetCollection"] + )[] + | null + /** Format */ + format: string + /** Format Source */ + format_source: string | null + /** Hidden */ + hidden: boolean + /** Label */ + label: string | null + /** Metadata Source */ + metadata_source: string | null + /** Name */ + name: string + /** + * Type + * @constant + * @enum {string} + */ + type: "data" + } + /** ToolOutputFloat */ + ToolOutputFloat: { + /** Hidden */ + hidden: boolean + /** Label */ + label: string | null + /** Name */ + name: string + /** + * Type + * @constant + * @enum {string} + */ + type: "float" + } + /** ToolOutputInteger */ + ToolOutputInteger: { + /** Hidden */ + hidden: boolean + /** Label */ + label: string | null + /** Name */ + name: string + /** + * Type + * @constant + * @enum {string} + */ + type: "integer" + } + /** ToolOutputText */ + ToolOutputText: { + /** Hidden */ + hidden: boolean + /** Label */ + label: string | null + /** Name */ + name: string + /** + * Type + * @constant + * @enum {string} + */ + type: "text" + } + /** ToolProvidedMetadataDatasetCollection */ + ToolProvidedMetadataDatasetCollection: { + /** Assign Primary Output */ + assign_primary_output: boolean + /** Directory */ + directory: string | null + /** + * Discover Via + * @constant + * @enum {string} + */ + discover_via: "tool_provided_metadata" + /** Format */ + format: string | null + /** Match Relative Path */ + match_relative_path: boolean + /** Recurse */ + recurse: boolean + /** Visible */ + visible: boolean + } /** ToolVersion */ ToolVersion: { /** @@ -1023,6 +2050,13 @@ export interface components { /** Version Major */ version_major: string } + /** XrefDict */ + XrefDict: { + /** Reftype */ + reftype: string + /** Value */ + value: string + } } responses: never parameters: never @@ -2005,6 +3039,71 @@ export interface operations { } } } + /** Return Galaxy's meta model description of the tool's inputs */ + tools__parameter_model: { + parameters: { + path: { + /** @description See also https://ga4gh.github.io/tool-registry-service-schemas/DataModel/#trs-tool-and-trs-tool-version-ids */ + tool_id: string + /** @description The full version string defined on the Galaxy tool wrapper. */ + tool_version: string + } + } + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["ParsedTool"] + } + } + /** @description Request Error */ + "4XX": { + content: { + "application/json": components["schemas"]["MessageExceptionModel"] + } + } + /** @description Server Error */ + "5XX": { + content: { + "application/json": components["schemas"]["MessageExceptionModel"] + } + } + } + } + /** + * Return a JSON schema description of the tool's inputs for the tool request API that will be added to Galaxy at some point + * @description The tool request schema includes validation of map/reduce concepts that can be consumed by the tool execution API and not just the request for a single execution. + */ + tools__parameter_request_model: { + parameters: { + path: { + /** @description See also https://ga4gh.github.io/tool-registry-service-schemas/DataModel/#trs-tool-and-trs-tool-version-ids */ + tool_id: string + /** @description The full version string defined on the Galaxy tool wrapper. */ + tool_version: string + } + } + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": unknown + } + } + /** @description Request Error */ + "4XX": { + content: { + "application/json": components["schemas"]["MessageExceptionModel"] + } + } + /** @description Server Error */ + "5XX": { + content: { + "application/json": components["schemas"]["MessageExceptionModel"] + } + } + } + } /** * Index * @description index users