From c0ce079b3eb84fe17c222cee0b461be47dc2f965 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Fri, 28 Feb 2025 14:52:56 -0500 Subject: [PATCH 01/23] #2 start on json doc --- .gitignore | 3 +++ docs/README.md | 6 ++++++ docs/accel.json | 23 +++++++++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 docs/README.md create mode 100644 docs/accel.json diff --git a/.gitignore b/.gitignore index 15201ac..924b9fb 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,6 @@ cython_debug/ # PyPI configuration file .pypirc + +.idea +.DS_Store diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..1e5b899 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,6 @@ +# Accelerator Data Model +## Version: 0.0.1-alpha + +## Description + +This accel.json file, and the accompanying JSON schema, represent the core data model for Accelerator. \ No newline at end of file diff --git a/docs/accel.json b/docs/accel.json new file mode 100644 index 0000000..3380363 --- /dev/null +++ b/docs/accel.json @@ -0,0 +1,23 @@ +{ + "submission": { + "submitter_name": null, + "submitter_email": null, + "submitter_comment": null + }, + "data": { + + }, + "technical_metadata": { + "created": null, + "modified": null, + "original_source": null, + "original_source_link": null, + "history": [ + { + "timestamp": null, + "msg": null + } + ] + + } +} \ No newline at end of file From 8ac9acb7b8b3dbcb2420757024cf9c7cd2e7072d Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Fri, 28 Feb 2025 15:02:34 -0500 Subject: [PATCH 02/23] Create jsonlint.yml --- .github/workflows/jsonlint.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/jsonlint.yml diff --git a/.github/workflows/jsonlint.yml b/.github/workflows/jsonlint.yml new file mode 100644 index 0000000..714b5da --- /dev/null +++ b/.github/workflows/jsonlint.yml @@ -0,0 +1,24 @@ +--- +name: Jsonlint-CI +on: + push: + branches: + - main + pull_request: + branches: + - main +jobs: + test: + runs-on: ubuntu-latest + container: + image: ixxel/unit-tests-alpine:latest + steps: + - name: Git clone repo + uses: actions/checkout@v2 + - name: Jsonlint scan + uses: ixxeL-actions/jsonlint@main + with: + extensions: '"json,JSON"' + environment: '""' + fix: 'true' + working-dir: ./test.json From e841fd78ece90d8696eb4ef8fc239ade11f06612 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Mon, 3 Mar 2025 08:05:41 -0500 Subject: [PATCH 03/23] add json lint action --- .github/workflows/jsonlint.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/jsonlint.yml b/.github/workflows/jsonlint.yml index 714b5da..659b69f 100644 --- a/.github/workflows/jsonlint.yml +++ b/.github/workflows/jsonlint.yml @@ -4,9 +4,12 @@ on: push: branches: - main + - develop + - feature/** pull_request: branches: - main + - develop jobs: test: runs-on: ubuntu-latest From 7218d573e81f70d362e7e10d8b857906f12406b4 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Mon, 3 Mar 2025 08:08:15 -0500 Subject: [PATCH 04/23] add json lint action --- .github/workflows/jsonlint.yml | 2 +- {docs => schema}/README.md | 0 {docs => schema}/accel.json | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {docs => schema}/README.md (100%) rename {docs => schema}/accel.json (100%) diff --git a/.github/workflows/jsonlint.yml b/.github/workflows/jsonlint.yml index 659b69f..d4c03e5 100644 --- a/.github/workflows/jsonlint.yml +++ b/.github/workflows/jsonlint.yml @@ -24,4 +24,4 @@ jobs: extensions: '"json,JSON"' environment: '""' fix: 'true' - working-dir: ./test.json + working-dir: ./schema/accel.json diff --git a/docs/README.md b/schema/README.md similarity index 100% rename from docs/README.md rename to schema/README.md diff --git a/docs/accel.json b/schema/accel.json similarity index 100% rename from docs/accel.json rename to schema/accel.json From 8bb5b1952014bda478e96ed99d662092f9c9ec7e Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Mon, 3 Mar 2025 08:09:49 -0500 Subject: [PATCH 05/23] target action to branch --- .github/workflows/jsonlint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/jsonlint.yml b/.github/workflows/jsonlint.yml index d4c03e5..814db31 100644 --- a/.github/workflows/jsonlint.yml +++ b/.github/workflows/jsonlint.yml @@ -6,6 +6,7 @@ on: - main - develop - feature/** + - 2-make-dummy-json-doc-and-starter-schema pull_request: branches: - main From 4f1004fe9cd58b9601c57a0bfa12af48de1e8094 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Mon, 3 Mar 2025 12:21:02 -0500 Subject: [PATCH 06/23] #5 adding slot for actions --- .github/actions/.gitinclude | 0 .github/workflows/jsonlint.yml | 1 - 2 files changed, 1 deletion(-) create mode 100644 .github/actions/.gitinclude diff --git a/.github/actions/.gitinclude b/.github/actions/.gitinclude new file mode 100644 index 0000000..e69de29 diff --git a/.github/workflows/jsonlint.yml b/.github/workflows/jsonlint.yml index 814db31..d4c03e5 100644 --- a/.github/workflows/jsonlint.yml +++ b/.github/workflows/jsonlint.yml @@ -6,7 +6,6 @@ on: - main - develop - feature/** - - 2-make-dummy-json-doc-and-starter-schema pull_request: branches: - main From db619b65533fcc5a124ef06c3a42f26aeb36d8c4 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Mon, 3 Mar 2025 12:46:49 -0500 Subject: [PATCH 07/23] #5 model --- schema/README.md | 9 ++++++++- schema/accel.json | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/schema/README.md b/schema/README.md index 1e5b899..223dc60 100644 --- a/schema/README.md +++ b/schema/README.md @@ -3,4 +3,11 @@ ## Description -This accel.json file, and the accompanying JSON schema, represent the core data model for Accelerator. \ No newline at end of file +This accel.json file, and the accompanying JSON schema, represent the core data model for Accelerator. + + +## Development process + +The initial accel.json file is derived from the Excel model of the 1.5 CHORDS model +found here: https://nih.sharepoint.com/:f:/r/sites/NIH-DataAcceleratorProgram/Shared%20Documents/General/Design/term%20mappings?csf=1&web=1&e=1JTZT1 + diff --git a/schema/accel.json b/schema/accel.json index 3380363..10ca84c 100644 --- a/schema/accel.json +++ b/schema/accel.json @@ -5,6 +5,19 @@ "submitter_comment": null }, "data": { + "program": { + + }, + "project": { + + }, + "resource": { + + }, + "data_resource": { + + } + }, "technical_metadata": { From ce8c9fcf050233c2878707f729c02d4254a2dfec Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Tue, 4 Mar 2025 15:42:32 -0500 Subject: [PATCH 08/23] #5 add schema --- schema/accel-schema.json | 473 +++++++++++++++++++++++++++++++++++++++ schema/accel.json | 112 +++++++-- 2 files changed, 568 insertions(+), 17 deletions(-) create mode 100644 schema/accel-schema.json diff --git a/schema/accel-schema.json b/schema/accel-schema.json new file mode 100644 index 0000000..93abfca --- /dev/null +++ b/schema/accel-schema.json @@ -0,0 +1,473 @@ +{ + "$schema": "http://json-schema.org/2020-12/schema#", + "$id": "http://www.niehs.nih.gov/schemas/accelerator.json", + "$ref": "#/definitions/Accelerator", + "description": "Data model for accelerator metadata", + "definitions": { + "Accelerator": { + "type": "object", + "additionalProperties": false, + "properties": { + "submission": { + "$ref": "#/definitions/Submission" + }, + "data": { + "$ref": "#/definitions/Data" + }, + "technical_metadata": { + "$ref": "#/definitions/TechnicalMetadata" + } + }, + "required": [ + "data", + "submission" + ], + "title": "Accelerator" + }, + "Data": { + "type": "object", + "additionalProperties": false, + "properties": { + "program": { + "$ref": "#/definitions/Program" + }, + "project": { + "$ref": "#/definitions/Project" + }, + "resource": { + "$ref": "#/definitions/Resource" + }, + "data_resource": { + "$ref": "#/definitions/DataResource" + }, + "temporal_data": { + "$ref": "#/definitions/TemporalData" + }, + "population_data": { + "$ref": "#/definitions/PopulationData" + }, + "geospatial_data": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "array", + "items": {} + }, + { + "type": "null" + } + ] + } + } + }, + "required": [ + "data_resource", + "geospatial_data", + "population_data", + "program", + "project", + "resource", + "temporal_data" + ], + "title": "Data" + }, + "DataResource": { + "type": "object", + "additionalProperties": false, + "properties": { + "exposure_media": { + "type": "array", + "items": {} + }, + "measures": { + "type": "array", + "items": {} + }, + "measures_other": { + "type": "array", + "items": {} + }, + "measurement_method": { + "type": "null" + }, + "measurement_method_other": { + "type": "null" + }, + "time_extent_start": { + "type": "null" + }, + "time_extent_end": { + "type": "null" + }, + "time_available_comment": { + "type": "null" + }, + "data_formats": { + "type": "array", + "items": {} + }, + "data_location": { + "type": "array", + "items": { + "$ref": "#/definitions/DataLocation" + } + } + }, + "required": [ + "data_formats", + "data_location", + "exposure_media", + "measurement_method", + "measurement_method_other", + "measures", + "measures_other", + "time_available_comment", + "time_extent_end", + "time_extent_start" + ], + "title": "DataResource" + }, + "DataLocation": { + "type": "object", + "additionalProperties": false, + "properties": { + "data_location_text": { + "type": "null" + }, + "data_location_link": { + "type": "null" + } + }, + "required": [ + "data_location_link", + "data_location_text" + ], + "title": "DataLocation" + }, + "PopulationData": { + "type": "object", + "additionalProperties": false, + "properties": { + "individual_level": { + "type": "boolean" + }, + "population_studies": { + "type": "array", + "items": {} + }, + "population_studies_other": { + "type": "array", + "items": {} + }, + "linkable_encounters": { + "type": "boolean" + }, + "biospecimens_from_humans": { + "type": "boolean" + }, + "biospecimens_type": { + "type": "array", + "items": {} + } + }, + "required": [ + "biospecimens_from_humans", + "biospecimens_type", + "individual_level", + "linkable_encounters", + "population_studies", + "population_studies_other" + ], + "title": "PopulationData" + }, + "Program": { + "type": "object", + "additionalProperties": false, + "properties": { + "program_name": { + "type": "null" + }, + "preferred_label": { + "type": "null" + } + }, + "required": [ + "preferred_label", + "program_name" + ], + "title": "Program" + }, + "Project": { + "type": "object", + "additionalProperties": false, + "properties": { + "project_code": { + "type": "null" + }, + "project_name": { + "type": "null" + }, + "project_short_name": { + "type": "null" + }, + "project_sponsor": { + "type": "array", + "items": {} + }, + "project_sponsor_other": { + "type": "array", + "items": {} + }, + "project_sponsor_type": { + "type": "array", + "items": {} + }, + "project_sponsor_type_other": { + "type": "array", + "items": {} + } + }, + "required": [ + "project_code", + "project_name", + "project_short_name", + "project_sponsor", + "project_sponsor_other", + "project_sponsor_type", + "project_sponsor_type_other" + ], + "title": "Project" + }, + "Resource": { + "type": "object", + "additionalProperties": false, + "properties": { + "resource_name": { + "type": "null" + }, + "resource_version": { + "type": "null" + }, + "resource_short_name": { + "type": "null" + }, + "resource_type": { + "type": "null" + }, + "resource_url": { + "type": "null" + }, + "resource_description": { + "type": "null" + }, + "resource_domain": { + "type": "array", + "items": {} + }, + "resource_domain_other": { + "type": "array", + "items": {} + }, + "resource_keywords": { + "type": "array", + "items": {} + }, + "resource_access_type": { + "type": "null" + }, + "resource_reference": { + "type": "array", + "items": { + "$ref": "#/definitions/ResourceReference" + } + }, + "resource_use_agreement": { + "type": "array", + "items": { + "$ref": "#/definitions/ResourceUseAgreement" + } + }, + "publication": { + "type": "array", + "items": { + "$ref": "#/definitions/Publication" + } + }, + "is_static": { + "type": "boolean" + }, + "comments": { + "type": "null" + } + }, + "required": [ + "comments", + "is_static", + "publication", + "resource_access_type", + "resource_description", + "resource_domain", + "resource_domain_other", + "resource_keywords", + "resource_name", + "resource_reference", + "resource_short_name", + "resource_type", + "resource_url", + "resource_use_agreement", + "resource_version" + ], + "title": "Resource" + }, + "Publication": { + "type": "object", + "additionalProperties": false, + "properties": { + "citation": { + "type": "null" + }, + "citation_link": { + "type": "null" + } + }, + "required": [ + "citation", + "citation_link" + ], + "title": "Publication" + }, + "ResourceReference": { + "type": "object", + "additionalProperties": false, + "properties": { + "resource_reference_text": { + "type": "null" + }, + "resource_reference_link": { + "type": "null" + } + }, + "required": [ + "resource_reference_link", + "resource_reference_text" + ], + "title": "ResourceReference" + }, + "ResourceUseAgreement": { + "type": "object", + "additionalProperties": false, + "properties": { + "resource_use_agreement_text": { + "type": "null" + }, + "resource_use_agreement_link": { + "type": "null" + } + }, + "required": [ + "resource_use_agreement_link", + "resource_use_agreement_text" + ], + "title": "ResourceUseAgreement" + }, + "TemporalData": { + "type": "object", + "additionalProperties": false, + "properties": { + "temporal_resolution": { + "type": "array", + "items": {} + }, + "temporal_resolution_other": { + "type": "array", + "items": {} + }, + "temporal_resolution_all_available": { + "type": "array", + "items": {} + }, + "temporal_resolution_comment": { + "type": "null" + } + }, + "required": [ + "temporal_resolution", + "temporal_resolution_all_available", + "temporal_resolution_comment", + "temporal_resolution_other" + ], + "title": "TemporalData" + }, + "Submission": { + "type": "object", + "additionalProperties": false, + "properties": { + "submitter_name": { + "type": "null" + }, + "submitter_email": { + "type": "null" + }, + "submitter_comment": { + "type": "null" + } + }, + "required": [ + "submitter_comment", + "submitter_email", + "submitter_name" + ], + "title": "Submission" + }, + "TechnicalMetadata": { + "type": "object", + "additionalProperties": false, + "properties": { + "created": { + "type": "null" + }, + "modified": { + "type": "null" + }, + "original_source": { + "type": "null" + }, + "original_source_link": { + "type": "null" + }, + "history": { + "type": "array", + "items": { + "$ref": "#/definitions/History" + } + } + }, + "required": [ + "created", + "history", + "modified", + "original_source", + "original_source_link" + ], + "title": "TechnicalMetadata" + }, + "History": { + "type": "object", + "additionalProperties": false, + "properties": { + "timestamp": { + "type": "null" + }, + "msg": { + "type": "null" + } + }, + "required": [ + "msg", + "timestamp" + ], + "title": "History" + } + } +} diff --git a/schema/accel.json b/schema/accel.json index 10ca84c..fcbf7d7 100644 --- a/schema/accel.json +++ b/schema/accel.json @@ -6,31 +6,109 @@ }, "data": { "program": { - + "program_name": null, + "preferred_label": null }, "project": { - + "project_code": null, + "project_name": null, + "project_short_name": null, + "project_sponsor": [], + "project_sponsor_other": [], + "project_sponsor_type": [], + "project_sponsor_type_other": [] }, "resource": { - + "resource_name": null, + "resource_version": null, + "resource_short_name": null, + "resource_type": null, + "resource_url": null, + "resource_description": null, + "resource_domain": [], + "resource_domain_other": [], + "resource_keywords": [], + "resource_access_type": null, + "resource_reference": [ + { + "resource_reference_text": null, + "resource_reference_link": null + } + ], + "resource_use_agreement": [ + { + "resource_use_agreement_text": null, + "resource_use_agreement_link": null + } + ], + "publication": [ + { + "citation": null, + "citation_link": null + } + ], + "is_static": false, + "comments": null }, "data_resource": { - + "exposure_media": [], + "measures": [], + "measures_other": [], + "measurement_method": null, + "measurement_method_other": null, + "time_extent_start": null, + "time_extent_end": null, + "time_available_comment": null, + "data_formats": [], + "data_location": [ + { + "data_location_text": null, + "data_location_link": null + } + ] + }, + "temporal_data": { + "temporal_resolution": [], + "temporal_resolution_other": [], + "temporal_resolution_all_available": [], + "temporal_resolution_comment": null + }, + "population_data": { + "individual_level": false, + "population_studies": [], + "population_studies_other": [], + "linkable_encounters": false, + "biospecimens_from_humans": false, + "biospecimens_type": [] + }, + "geospatial_data": { + "spatial_resolution": [], + "spatial_resolution_other": [], + "spatial_resolution_all_available": [], + "spatial_resolution_comment": null, + "spatial_coverage": [], + "spatial_coverage_other": [], + "spatial_bounding_box": [], + "geometry_type": [], + "geometry_source": [], + "geometry_source_other": [], + "model_methods": [], + "model_methods_other": [], + "exposure_media": [], + "geographic_feature": [], + "geographic_feature_other": [] } - - }, "technical_metadata": { - "created": null, - "modified": null, - "original_source": null, - "original_source_link": null, - "history": [ - { - "timestamp": null, - "msg": null - } - ] - + "created": null, + "modified": null, + "original_source": null, + "original_source_link": null, + "history": [ + { + "timestamp": null, + "msg": null + } + ] } } \ No newline at end of file From 8e1ef036f75eea73a9496ddbf075d7c290663f9f Mon Sep 17 00:00:00 2001 From: pateldes Date: Tue, 4 Mar 2025 15:55:11 -0500 Subject: [PATCH 09/23] #1 add basics structure --- accelerator_core/__init__.py | 0 accelerator_core/accession.py | 29 ++++++++++++++++++ accelerator_core/crosswalk.py | 10 +++++++ accelerator_core/db/__init__.py | 0 accelerator_core/db/connector.py | 0 accelerator_core/db/models.py | 0 accelerator_core/dissemination.py | 18 +++++++++++ accelerator_core/utils/__init__.py | 0 accelerator_core/utils/config.py | 0 accelerator_core/utils/logger.py | 11 +++++++ docs/index.md | 48 ++++++++++++++++++++++++++++++ pyproject.toml | 29 ++++++++++++++++++ setup.py | 20 +++++++++++++ tests/test_accession.py | 10 +++++++ tests/test_crosswalk.py | 10 +++++++ tests/test_db.py | 10 +++++++ tests/test_dissemination.py | 10 +++++++ 17 files changed, 205 insertions(+) create mode 100644 accelerator_core/__init__.py create mode 100644 accelerator_core/accession.py create mode 100644 accelerator_core/crosswalk.py create mode 100644 accelerator_core/db/__init__.py create mode 100644 accelerator_core/db/connector.py create mode 100644 accelerator_core/db/models.py create mode 100644 accelerator_core/dissemination.py create mode 100644 accelerator_core/utils/__init__.py create mode 100644 accelerator_core/utils/config.py create mode 100644 accelerator_core/utils/logger.py create mode 100644 docs/index.md create mode 100644 pyproject.toml create mode 100644 setup.py create mode 100644 tests/test_accession.py create mode 100644 tests/test_crosswalk.py create mode 100644 tests/test_db.py create mode 100644 tests/test_dissemination.py diff --git a/accelerator_core/__init__.py b/accelerator_core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/accelerator_core/accession.py b/accelerator_core/accession.py new file mode 100644 index 0000000..0b57ceb --- /dev/null +++ b/accelerator_core/accession.py @@ -0,0 +1,29 @@ +from accelerator_core.crosswalk import Crosswalk + + +class Accession: + """Handles validation and CRUD operations for metadata records.""" + + def __init__(self, data: dict): + """Initialize Accession with validated data.""" + self.data = data + + def validate(self) -> bool: + """Validate JSON output from Crosswalk.""" + pass + + def create(self) -> str: + """Create a new record in the database.""" + pass + + def read(self, record_id: str) -> dict: + """Retrieve a record from the database.""" + pass + + def update(self, record_id: str, new_data: dict) -> bool: + """Update an existing record.""" + pass + + def delete(self, record_id: str) -> bool: + """Delete a record from the database.""" + pass diff --git a/accelerator_core/crosswalk.py b/accelerator_core/crosswalk.py new file mode 100644 index 0000000..7688e56 --- /dev/null +++ b/accelerator_core/crosswalk.py @@ -0,0 +1,10 @@ +from abc import ABC, abstractmethod + + +class Crosswalk(ABC): + """Abstract superclass for mapping raw data to a structured JSON format.""" + + @abstractmethod + def transform(self, raw_data: dict) -> dict: + """Convert raw data into a standardized format.""" + pass diff --git a/accelerator_core/db/__init__.py b/accelerator_core/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/accelerator_core/db/connector.py b/accelerator_core/db/connector.py new file mode 100644 index 0000000..e69de29 diff --git a/accelerator_core/db/models.py b/accelerator_core/db/models.py new file mode 100644 index 0000000..e69de29 diff --git a/accelerator_core/dissemination.py b/accelerator_core/dissemination.py new file mode 100644 index 0000000..44fc891 --- /dev/null +++ b/accelerator_core/dissemination.py @@ -0,0 +1,18 @@ +class Dissemination: + """Retrieves data from the database and transforms it into a JSON document for endpoint systems.""" + + def __init__(self, record_id: str): + """Initialize Dissemination with a record ID.""" + self.record_id = record_id + + def fetch_data(self) -> dict: + """Retrieve data from the database.""" + pass + + def format_for_endpoint(self, system: str) -> dict: + """Convert data into a format required by a specific endpoint (CHORDS, Navigator, CEDAR).""" + pass + + def export(self, system: str) -> bool: + """Send the formatted data to the respective system.""" + pass diff --git a/accelerator_core/utils/__init__.py b/accelerator_core/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/accelerator_core/utils/config.py b/accelerator_core/utils/config.py new file mode 100644 index 0000000..e69de29 diff --git a/accelerator_core/utils/logger.py b/accelerator_core/utils/logger.py new file mode 100644 index 0000000..725b168 --- /dev/null +++ b/accelerator_core/utils/logger.py @@ -0,0 +1,11 @@ +import logging + + +def setup_logger(name: str): + """Setup and return a logger instance.""" + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + ch = logging.StreamHandler() + ch.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) + logger.addHandler(ch) + return logger diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..474ab51 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,48 @@ +# Main documentation file + + + +## Project structure overview +``` +accelerator-core/ +│── accelerator_core/ # Main package directory +│ ├── __init__.py # Makes this a Python package +│ ├── accession.py # Accession class (CRUD operations, validation) +│ ├── dissemination.py # Dissemination class (data retrieval, JSON transformation) +│ ├── crosswalk.py # Superclass for data mapping +│ ├── db/ # Database interaction module +│ │ ├── __init__.py +│ │ ├── models.py # Database models +│ │ ├── connector.py # Database connection logic +│ ├── utils/ # Utility functions (common helpers) +│ │ ├── __init__.py +│ │ ├── logger.py # Logging setup +│ │ ├── config.py # Configuration management +│ +├── tests/ # Unit tests directory +│ ├── __init__.py +│ ├── test_accession.py # Unit tests for accession module +│ ├── test_dissemination.py # Unit tests for dissemination module +│ ├── test_crosswalk.py # Unit tests for crosswalk superclass +│ ├── test_db.py # Tests for database interaction +│ +├── docs/ # Documentation +│ ├── index.md # Main documentation file +│ ├── api_reference.md # API reference +│ ├── usage_guide.md # Usage guide +│ +├── .gitignore # Git ignore file +├── pyproject.toml # Build system configuration (PEP 517) +├── setup.py # Legacy setup script for packaging +├── setup.cfg # Configuration for setuptools +├── README.md # Project overview and usage +├── LICENSE # License file +``` + +### Key Features: +* accelerator_core/ contains the core modules: accession.py, dissemination.py, and crosswalk.py. +* db/ handles database interactions. +* utils/ holds helper functions like logging and config management. +* tests/ includes unit tests to validate functionality. +* docs/ provides API reference and usage documentation. +* pyproject.toml, setup.py, and setup.cfg enable packaging and distribution via pip. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d5a288d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "accelerator-core" +version = "0.1.0" +description = "Core libraries for the accelerator metadata backbone" +readme = "README.md" +authors = [{ name = "Your Name", email = "your.email@example.com" }] +license = { file = "LICENSE" } +keywords = ["metadata", "data-processing", "accelerator"] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", +] +requires-python = ">=3.8" +dependencies = [ + +] + +[project.urls] +Homepage = "https://github.com/yourusername/accelerator-core" +Documentation = "https://github.com/yourusername/accelerator-core/wiki" +Repository = "https://github.com/yourusername/accelerator-core" + +[tool.setuptools.packages.find] +where = ["accelerator_core"] diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..5a562ff --- /dev/null +++ b/setup.py @@ -0,0 +1,20 @@ +from setuptools import setup, find_packages + +setup( + name="accelerator-core", + version="0.1.0", + description="Core libraries for the accelerator metadata backbone", + author="Your Name", + author_email="your.email@example.com", + url="https://github.com/yourusername/accelerator-core", + packages=find_packages(), + install_requires=[ + ], + license="BSD 3-Clause", + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + ], + python_requires=">=3.9", +) \ No newline at end of file diff --git a/tests/test_accession.py b/tests/test_accession.py new file mode 100644 index 0000000..493b0f1 --- /dev/null +++ b/tests/test_accession.py @@ -0,0 +1,10 @@ +import unittest + + +class MyTestCase(unittest.TestCase): + def test_something(self): + self.assertEqual(True, False) # add assertion here + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_crosswalk.py b/tests/test_crosswalk.py new file mode 100644 index 0000000..493b0f1 --- /dev/null +++ b/tests/test_crosswalk.py @@ -0,0 +1,10 @@ +import unittest + + +class MyTestCase(unittest.TestCase): + def test_something(self): + self.assertEqual(True, False) # add assertion here + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..493b0f1 --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,10 @@ +import unittest + + +class MyTestCase(unittest.TestCase): + def test_something(self): + self.assertEqual(True, False) # add assertion here + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_dissemination.py b/tests/test_dissemination.py new file mode 100644 index 0000000..493b0f1 --- /dev/null +++ b/tests/test_dissemination.py @@ -0,0 +1,10 @@ +import unittest + + +class MyTestCase(unittest.TestCase): + def test_something(self): + self.assertEqual(True, False) # add assertion here + + +if __name__ == '__main__': + unittest.main() From 6657ec71fc2a025f0c554b91ef6bdf1f76d03a0d Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 10:58:01 -0500 Subject: [PATCH 10/23] #5 schema val and tests --- {schema => accelerator_core/schema}/README.md | 0 .../{utils => schema}/__init__.py | 0 .../schema/accel-schema-v1.0.0.json | 0 .../schema}/accel.json | 0 accelerator_core/utils/config.py | 21 +++++++++ accelerator_core/utils/schema_tools.py | 45 +++++++++++++++++++ requirements.txt | 3 ++ tests/test_schema_tools.py | 25 +++++++++++ 8 files changed, 94 insertions(+) rename {schema => accelerator_core/schema}/README.md (100%) rename accelerator_core/{utils => schema}/__init__.py (100%) rename schema/accel-schema.json => accelerator_core/schema/accel-schema-v1.0.0.json (100%) rename {schema => accelerator_core/schema}/accel.json (100%) create mode 100644 accelerator_core/utils/schema_tools.py create mode 100644 requirements.txt create mode 100644 tests/test_schema_tools.py diff --git a/schema/README.md b/accelerator_core/schema/README.md similarity index 100% rename from schema/README.md rename to accelerator_core/schema/README.md diff --git a/accelerator_core/utils/__init__.py b/accelerator_core/schema/__init__.py similarity index 100% rename from accelerator_core/utils/__init__.py rename to accelerator_core/schema/__init__.py diff --git a/schema/accel-schema.json b/accelerator_core/schema/accel-schema-v1.0.0.json similarity index 100% rename from schema/accel-schema.json rename to accelerator_core/schema/accel-schema-v1.0.0.json diff --git a/schema/accel.json b/accelerator_core/schema/accel.json similarity index 100% rename from schema/accel.json rename to accelerator_core/schema/accel.json diff --git a/accelerator_core/utils/config.py b/accelerator_core/utils/config.py index e69de29..955c395 100644 --- a/accelerator_core/utils/config.py +++ b/accelerator_core/utils/config.py @@ -0,0 +1,21 @@ +""" +Utilities related to configuration, resources and environment variables. +""" +import importlib.resources +from pathlib import Path +from accelerator_core.utils.logger import setup_logger +import json + +logger = setup_logger('accelerator') + +def determine_resource_path(resource_package, resource_name) -> Path: + """ + Given a package and resource name, get the given program resource + :param resource_package: package where resources are located + :param resource_name: name of the resource in the package + :return: Path object to the resource + """ + + with importlib.resources.path(resource_package, resource_name) as fspath: + logger.debug(f"resource path:{fspath}") + return fspath \ No newline at end of file diff --git a/accelerator_core/utils/schema_tools.py b/accelerator_core/utils/schema_tools.py new file mode 100644 index 0000000..0abcaf4 --- /dev/null +++ b/accelerator_core/utils/schema_tools.py @@ -0,0 +1,45 @@ +import importlib.resources + +from accelerator_core.utils.logger import setup_logger +import accelerator_core.schema +from accelerator_core.utils.config import determine_resource_path +import json +import jsonschema + +logger = setup_logger('accelerator') + +CURRENT_ACCEL_SCHEMA_VERSION = '1.0.0' +CURRENT_JSON_SCHEMA_VERSION = '2020-12' + +def read_current_schema(schema_version: str = CURRENT_ACCEL_SCHEMA_VERSION): + """ + Read the current JSON schema + :param schema_version: n.n.n schema version (defaults to most current) + :return: json object representing schema + """ + + schema_name = f"accel-schema-v{schema_version}.json" + + with determine_resource_path(accelerator_core.schema, schema_name) as fspath: + logger.debug(f"resource path:{fspath}") + + with open(fspath) as json_data: + d = json.load(json_data) + return d + +def validate_json_against_schema(json_doc, schema_version: str = CURRENT_ACCEL_SCHEMA_VERSION)-> bool: + """ + validate the given json (as a json dict) + :param json_doc: dict with json to validate + :param schema_version: version of accel schema to validate against, as a version string: 1.0.0 + :return: bool is True if valid + """ + + json_schema_doc = read_current_schema(schema_version) + try: + jsonschema.validate(json_doc, json_schema_doc) + except jsonschema.exceptions.ValidationError as e: + logger.error(f"invalid json document {e}") + return False + + return True diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8fba323 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +setuptools~=65.6.3 +jsonschema~=4.23.0 +pre-commit~=4.1.0 \ No newline at end of file diff --git a/tests/test_schema_tools.py b/tests/test_schema_tools.py new file mode 100644 index 0000000..72c597d --- /dev/null +++ b/tests/test_schema_tools.py @@ -0,0 +1,25 @@ +import json +import unittest + +import accelerator_core +from accelerator_core.utils.config import determine_resource_path +from accelerator_core.utils.schema_tools import read_current_schema, validate_json_against_schema + + +class TestSchemaTools(unittest.TestCase): + def test_read_current_schema(self): + schema_json = read_current_schema() + self.assertIsNotNone(schema_json, "did not schema ") + self.assertIsInstance(schema_json, dict) + + + def test_validate_schema(self): + json_path = determine_resource_path(accelerator_core.schema, "accel.json") + with open(json_path) as json_data: + d = json.load(json_data) + + actual = validate_json_against_schema(d) + self.assertTrue(actual, 'Did not validate json against schema') + +if __name__ == '__main__': + unittest.main() From 41e42ecc725bfb00012b41fc20298a0fc00c36a9 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 11:31:09 -0500 Subject: [PATCH 11/23] #5 add pre-commit hooks --- .pre-commit-config.yaml | 10 ++++++++++ README.md | 14 +++++++++++++- accelerator_core/schema/README.md | 3 +-- accelerator_core/schema/accel.json | 2 +- accelerator_core/utils/config.py | 2 +- requirements.txt | 2 +- setup.py | 2 +- 7 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..fd16ba2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files diff --git a/README.md b/README.md index 30a59c7..a589c3f 100644 --- a/README.md +++ b/README.md @@ -12,4 +12,16 @@ The project implements all of the core components (accession, dissemination) and Each source and disseination target should be developed in a separate accelerator-source-xxx or accelerator-dissemination-xxx repository - + +## Developer Notes + +This project uses pre-commit hooks to validate code, run tests, and accomplish other tasks. + +Loading the requirements.txt into your dev environment will install pre-commit, then you can set up the pre-commit +hooks by running: + +``` +pre-commit install +``` + +This should be the first thing you do when cloning this project diff --git a/accelerator_core/schema/README.md b/accelerator_core/schema/README.md index 223dc60..cbf299d 100644 --- a/accelerator_core/schema/README.md +++ b/accelerator_core/schema/README.md @@ -3,11 +3,10 @@ ## Description -This accel.json file, and the accompanying JSON schema, represent the core data model for Accelerator. +This accel.json file, and the accompanying JSON schema, represent the core data model for Accelerator. ## Development process The initial accel.json file is derived from the Excel model of the 1.5 CHORDS model found here: https://nih.sharepoint.com/:f:/r/sites/NIH-DataAcceleratorProgram/Shared%20Documents/General/Design/term%20mappings?csf=1&web=1&e=1JTZT1 - diff --git a/accelerator_core/schema/accel.json b/accelerator_core/schema/accel.json index fcbf7d7..3fdcb33 100644 --- a/accelerator_core/schema/accel.json +++ b/accelerator_core/schema/accel.json @@ -111,4 +111,4 @@ } ] } -} \ No newline at end of file +} diff --git a/accelerator_core/utils/config.py b/accelerator_core/utils/config.py index 955c395..3d5e8e4 100644 --- a/accelerator_core/utils/config.py +++ b/accelerator_core/utils/config.py @@ -18,4 +18,4 @@ def determine_resource_path(resource_package, resource_name) -> Path: with importlib.resources.path(resource_package, resource_name) as fspath: logger.debug(f"resource path:{fspath}") - return fspath \ No newline at end of file + return fspath diff --git a/requirements.txt b/requirements.txt index 8fba323..1530dde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ setuptools~=65.6.3 jsonschema~=4.23.0 -pre-commit~=4.1.0 \ No newline at end of file +pre-commit~=4.1.0 diff --git a/setup.py b/setup.py index 5a562ff..fa33a7a 100644 --- a/setup.py +++ b/setup.py @@ -17,4 +17,4 @@ "Operating System :: OS Independent", ], python_requires=">=3.9", -) \ No newline at end of file +) From 3616e2342cef6f52c4f16e9bd11ea67398a6ef4d Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 11:40:24 -0500 Subject: [PATCH 12/23] #5 add formatter as action and pre-commit --- .github/workflows/black.yml | 10 ++++++++++ .pre-commit-config.yaml | 5 +++++ README.md | 4 +++- requirements.txt | 1 + 4 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/black.yml diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml new file mode 100644 index 0000000..81e6a94 --- /dev/null +++ b/.github/workflows/black.yml @@ -0,0 +1,10 @@ +name: Lint + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: psf/black@stable diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fd16ba2..4d82889 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,3 +8,8 @@ repos: - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files +- repo: https://github.com/psf/black + rev: 25.1.0 + hooks: + - id: black + language_version: python3.11 diff --git a/README.md b/README.md index a589c3f..7b2dffd 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ Core libraries and classes for accelerator metadata backbone This project is the base python package and libraries for the core of the Accellerator Project +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + ![System Whiteboard](https://github.com/user-attachments/assets/2a2b07fa-bbed-454c-9050-73eccb7cbf6c) @@ -24,4 +26,4 @@ hooks by running: pre-commit install ``` -This should be the first thing you do when cloning this project +This should be the first thing you do when cloning this project. More docs on pre-commit are available [here](https://pre-commit.com/) diff --git a/requirements.txt b/requirements.txt index 1530dde..7c05283 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ setuptools~=65.6.3 jsonschema~=4.23.0 pre-commit~=4.1.0 +black~=25.1.0 From 812672816180c3b171c2ca1d8839017286def550 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 11:45:36 -0500 Subject: [PATCH 13/23] #5 add formatter as action and pre-commit --- .github/workflows/black.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 81e6a94..3d31a38 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -8,3 +8,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: psf/black@stable + with: + options: "--check --verbose" + src: "./accelerator_core" From edd9452310d77f6aaf8c589a6444faa177beef4f Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 12:01:24 -0500 Subject: [PATCH 14/23] #5 add formatter as action and pre-commit --- accelerator_core/utils/config.py | 4 +++- accelerator_core/utils/logger.py | 2 +- accelerator_core/utils/schema_tools.py | 12 ++++++++---- pyproject.toml | 3 +++ 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/accelerator_core/utils/config.py b/accelerator_core/utils/config.py index 3d5e8e4..4a1ac2f 100644 --- a/accelerator_core/utils/config.py +++ b/accelerator_core/utils/config.py @@ -1,12 +1,14 @@ """ Utilities related to configuration, resources and environment variables. """ + import importlib.resources from pathlib import Path from accelerator_core.utils.logger import setup_logger import json -logger = setup_logger('accelerator') +logger = setup_logger("accelerator") + def determine_resource_path(resource_package, resource_name) -> Path: """ diff --git a/accelerator_core/utils/logger.py b/accelerator_core/utils/logger.py index 725b168..fd387e9 100644 --- a/accelerator_core/utils/logger.py +++ b/accelerator_core/utils/logger.py @@ -6,6 +6,6 @@ def setup_logger(name: str): logger = logging.getLogger(name) logger.setLevel(logging.INFO) ch = logging.StreamHandler() - ch.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) + ch.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")) logger.addHandler(ch) return logger diff --git a/accelerator_core/utils/schema_tools.py b/accelerator_core/utils/schema_tools.py index 0abcaf4..13db0eb 100644 --- a/accelerator_core/utils/schema_tools.py +++ b/accelerator_core/utils/schema_tools.py @@ -6,10 +6,11 @@ import json import jsonschema -logger = setup_logger('accelerator') +logger = setup_logger("accelerator") + +CURRENT_ACCEL_SCHEMA_VERSION = "1.0.0" +CURRENT_JSON_SCHEMA_VERSION = "2020-12" -CURRENT_ACCEL_SCHEMA_VERSION = '1.0.0' -CURRENT_JSON_SCHEMA_VERSION = '2020-12' def read_current_schema(schema_version: str = CURRENT_ACCEL_SCHEMA_VERSION): """ @@ -27,7 +28,10 @@ def read_current_schema(schema_version: str = CURRENT_ACCEL_SCHEMA_VERSION): d = json.load(json_data) return d -def validate_json_against_schema(json_doc, schema_version: str = CURRENT_ACCEL_SCHEMA_VERSION)-> bool: + +def validate_json_against_schema( + json_doc, schema_version: str = CURRENT_ACCEL_SCHEMA_VERSION +) -> bool: """ validate the given json (as a json dict) :param json_doc: dict with json to validate diff --git a/pyproject.toml b/pyproject.toml index d5a288d..2ba7adc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,3 +27,6 @@ Repository = "https://github.com/yourusername/accelerator-core" [tool.setuptools.packages.find] where = ["accelerator_core"] + +[black] +include = "accelerator_core" From f7e602730d951128ad9f45f35b85531d2c20cfe7 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 12:26:00 -0500 Subject: [PATCH 15/23] #5 add pytest to github actions --- .github/workflows/jsonlint.yml | 17 +++++++++++++++++ requirements.txt | 1 + tests/__init__.py | 0 tests/context.py | 4 ++++ tests/test_accession.py | 4 ++-- tests/test_crosswalk.py | 4 ++-- tests/test_db.py | 4 ++-- tests/test_dissemination.py | 4 ++-- 8 files changed, 30 insertions(+), 8 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/context.py diff --git a/.github/workflows/jsonlint.yml b/.github/workflows/jsonlint.yml index d4c03e5..08afa8a 100644 --- a/.github/workflows/jsonlint.yml +++ b/.github/workflows/jsonlint.yml @@ -13,6 +13,9 @@ on: jobs: test: runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.11" ] container: image: ixxel/unit-tests-alpine:latest steps: @@ -25,3 +28,17 @@ jobs: environment: '""' fix: 'true' working-dir: ./schema/accel.json + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Test with pytest + run: | + coverage run -m pytest -v -s + - name: Generate Coverage Report + run: | + coverage report -m diff --git a/requirements.txt b/requirements.txt index 7c05283..24ef495 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ setuptools~=65.6.3 jsonschema~=4.23.0 pre-commit~=4.1.0 black~=25.1.0 +pytest~=8.3.5 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/context.py b/tests/context.py new file mode 100644 index 0000000..1406b77 --- /dev/null +++ b/tests/context.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) diff --git a/tests/test_accession.py b/tests/test_accession.py index 493b0f1..6fa47bb 100644 --- a/tests/test_accession.py +++ b/tests/test_accession.py @@ -3,8 +3,8 @@ class MyTestCase(unittest.TestCase): def test_something(self): - self.assertEqual(True, False) # add assertion here + self.assertEqual(True, True) # add assertion here -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_crosswalk.py b/tests/test_crosswalk.py index 493b0f1..6fa47bb 100644 --- a/tests/test_crosswalk.py +++ b/tests/test_crosswalk.py @@ -3,8 +3,8 @@ class MyTestCase(unittest.TestCase): def test_something(self): - self.assertEqual(True, False) # add assertion here + self.assertEqual(True, True) # add assertion here -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_db.py b/tests/test_db.py index 493b0f1..6fa47bb 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -3,8 +3,8 @@ class MyTestCase(unittest.TestCase): def test_something(self): - self.assertEqual(True, False) # add assertion here + self.assertEqual(True, True) # add assertion here -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_dissemination.py b/tests/test_dissemination.py index 493b0f1..6fa47bb 100644 --- a/tests/test_dissemination.py +++ b/tests/test_dissemination.py @@ -3,8 +3,8 @@ class MyTestCase(unittest.TestCase): def test_something(self): - self.assertEqual(True, False) # add assertion here + self.assertEqual(True, True) # add assertion here -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From 3112e1c3e75ad43e8caa67306c1fdcaff4feb510 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 12:33:11 -0500 Subject: [PATCH 16/23] #5 combine actions --- .github/workflows/black.yml | 13 --------- .github/workflows/jsonlint.yml | 44 ----------------------------- .github/workflows/lint-and-test.yml | 43 ++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 57 deletions(-) delete mode 100644 .github/workflows/black.yml delete mode 100644 .github/workflows/jsonlint.yml create mode 100644 .github/workflows/lint-and-test.yml diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml deleted file mode 100644 index 3d31a38..0000000 --- a/.github/workflows/black.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Lint - -on: [push, pull_request] - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: psf/black@stable - with: - options: "--check --verbose" - src: "./accelerator_core" diff --git a/.github/workflows/jsonlint.yml b/.github/workflows/jsonlint.yml deleted file mode 100644 index 08afa8a..0000000 --- a/.github/workflows/jsonlint.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -name: Jsonlint-CI -on: - push: - branches: - - main - - develop - - feature/** - pull_request: - branches: - - main - - develop -jobs: - test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ "3.11" ] - container: - image: ixxel/unit-tests-alpine:latest - steps: - - name: Git clone repo - uses: actions/checkout@v2 - - name: Jsonlint scan - uses: ixxeL-actions/jsonlint@main - with: - extensions: '"json,JSON"' - environment: '""' - fix: 'true' - working-dir: ./schema/accel.json - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Test with pytest - run: | - coverage run -m pytest -v -s - - name: Generate Coverage Report - run: | - coverage report -m diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml new file mode 100644 index 0000000..6d202b6 --- /dev/null +++ b/.github/workflows/lint-and-test.yml @@ -0,0 +1,43 @@ +--- +name: Jsonlint-CI +on: + push: + branches: + - main + - develop + - feature/** + pull_request: + branches: + - main + - develop +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: psf/black@stable + with: + options: "--check --verbose" + src: "./accelerator_core" + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.10" ] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Test with pytest + run: | + coverage run -m pytest -v -s + - name: Generate Coverage Report + run: | + coverage report -m From ea39ef63a9ba4b6a7ab18302980923032db98adf Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 12:36:21 -0500 Subject: [PATCH 17/23] #5 testing lint action --- accelerator_core/utils/schema_tools.py | 2 ++ requirements.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/accelerator_core/utils/schema_tools.py b/accelerator_core/utils/schema_tools.py index 13db0eb..95bd7cf 100644 --- a/accelerator_core/utils/schema_tools.py +++ b/accelerator_core/utils/schema_tools.py @@ -22,9 +22,11 @@ def read_current_schema(schema_version: str = CURRENT_ACCEL_SCHEMA_VERSION): schema_name = f"accel-schema-v{schema_version}.json" with determine_resource_path(accelerator_core.schema, schema_name) as fspath: + logger.debug(f"resource path:{fspath}") with open(fspath) as json_data: + d = json.load(json_data) return d diff --git a/requirements.txt b/requirements.txt index 24ef495..40b2975 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ jsonschema~=4.23.0 pre-commit~=4.1.0 black~=25.1.0 pytest~=8.3.5 +coverage~=7.6.12 From fd13048e1a9f5b100ec7b7ac584b787fbc271d13 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 12:38:02 -0500 Subject: [PATCH 18/23] #5 testing lint action --- accelerator_core/utils/schema_tools.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/accelerator_core/utils/schema_tools.py b/accelerator_core/utils/schema_tools.py index 95bd7cf..b96d987 100644 --- a/accelerator_core/utils/schema_tools.py +++ b/accelerator_core/utils/schema_tools.py @@ -22,11 +22,8 @@ def read_current_schema(schema_version: str = CURRENT_ACCEL_SCHEMA_VERSION): schema_name = f"accel-schema-v{schema_version}.json" with determine_resource_path(accelerator_core.schema, schema_name) as fspath: - logger.debug(f"resource path:{fspath}") - with open(fspath) as json_data: - d = json.load(json_data) return d From 085205d53ff626ede4cf832011e18d50bd6daad8 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 12:43:09 -0500 Subject: [PATCH 19/23] #5 status badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b2dffd..cd92f62 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Core libraries and classes for accelerator metadata backbone This project is the base python package and libraries for the core of the Accellerator Project [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) - +![status]((https://github.com/github/docs/actions/workflows/main.yml/badge.svg?event=push)) ![System Whiteboard](https://github.com/user-attachments/assets/2a2b07fa-bbed-454c-9050-73eccb7cbf6c) From 0c20c44377cd7391903c3ffcce4a0ecf93c44bd3 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 12:45:47 -0500 Subject: [PATCH 20/23] #5 status badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cd92f62..a8d8e64 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Core libraries and classes for accelerator metadata backbone This project is the base python package and libraries for the core of the Accellerator Project [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -![status]((https://github.com/github/docs/actions/workflows/main.yml/badge.svg?event=push)) +![status]((https://github.com/NIEHS/accelerator-core/actions/workflows/lint-and-test.yaml/badge.svg?event=push)) ![System Whiteboard](https://github.com/user-attachments/assets/2a2b07fa-bbed-454c-9050-73eccb7cbf6c) From 50af817b6d88dfee334525fde0051af8eee87452 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 12:47:49 -0500 Subject: [PATCH 21/23] #5 status badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a8d8e64..1717f6f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Core libraries and classes for accelerator metadata backbone This project is the base python package and libraries for the core of the Accellerator Project [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -![status]((https://github.com/NIEHS/accelerator-core/actions/workflows/lint-and-test.yaml/badge.svg?event=push)) +![status](https://github.com/NIEHS/accelerator-core/actions/workflows/lint-and-test.yaml/badge.svg?event=push) ![System Whiteboard](https://github.com/user-attachments/assets/2a2b07fa-bbed-454c-9050-73eccb7cbf6c) From 047bcb7c4933319e650e3dde74d45f62b89c915c Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 13:01:11 -0500 Subject: [PATCH 22/23] #5 status badge --- .github/workflows/lint-and-test.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 6d202b6..92782ad 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -1,5 +1,5 @@ --- -name: Jsonlint-CI +name: CI/CD on: push: branches: diff --git a/README.md b/README.md index 1717f6f..82abbec 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Core libraries and classes for accelerator metadata backbone ## Description -This project is the base python package and libraries for the core of the Accellerator Project +This project is the base python package and libraries for the core of the Accelerator Project [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) ![status](https://github.com/NIEHS/accelerator-core/actions/workflows/lint-and-test.yaml/badge.svg?event=push) From 86afa6a7c4707ce112bf59cc872df0035a401d08 Mon Sep 17 00:00:00 2001 From: Mike Conway Date: Wed, 5 Mar 2025 13:02:02 -0500 Subject: [PATCH 23/23] #5 combine actions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 82abbec..168069e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Core libraries and classes for accelerator metadata backbone This project is the base python package and libraries for the core of the Accelerator Project [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -![status](https://github.com/NIEHS/accelerator-core/actions/workflows/lint-and-test.yaml/badge.svg?event=push) +[![CI/CD](https://github.com/NIEHS/accelerator-core/actions/workflows/lint-and-test.yml/badge.svg)](https://github.com/NIEHS/accelerator-core/actions/workflows/lint-and-test.yml) ![System Whiteboard](https://github.com/user-attachments/assets/2a2b07fa-bbed-454c-9050-73eccb7cbf6c)