Merge pull request #205 from hed-standard/develop

Updated with latest fixes

.github/workflows/links.yaml

@@ -0,0 +1,24 @@
+---
+name: Lychee link checker
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  codespell:
+    name: Lychee link checker
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Link Checker
+        id: lychee
+        uses: lycheeverse/lychee-action@v1
+        with:
+          # Check all markdown, html and reStructuredText files in repo (default)
+          args: --base . --verbose --no-progress './**/*.md' './**/*.html' './**/*.rst'

.lycheeignore

@@ -0,0 +1,3 @@
+<!-- This file lists all links/regex to be ignored by lychee in the link checker -->
+(_anchor|-anchor)
+

docs/requirements.txt

@@ -1,29 +1,29 @@
-attrs>=21.4.0
-click>=8.1.3
-coverage>=6.3.2
-defusedxml>=0.7.1
-et-xmlfile>=1.1.0
-Flask>=3.0.3
-Flask-WTF>=1.2.1
-inflect>=5.5.2
-itsdangerous>=2.2.0
-jdcal>=1.4.1
-Jinja2>=3.1.2
+alabaster==0.7.16
+Babel==2.15.0
+certifi==2024.7.4
+charset-normalizer==3.3.2
+docutils==0.20.1
+idna==3.7
+imagesize==1.4.1
+Jinja2==3.1.4
+markdown-it-py==3.0.0
 MarkupSafe==2.1.5
-myst-parser>=0.17.0
-numpy>=1.20.3
-numpydoc>=1.3.1
-openpyxl>=3.0.9
-pandas>=1.3.5
-portalocker>=2.8.2
-Pygments>=2.12.0
-python-dateutil>=2.8.2
-pytz>=2022.1
-semantic_version>=2.9.0
-six==1.16.0
-Sphinx>=4,<8
-SphinxExtensions>=0.2.0
-sphinx_rtd_theme>=1.0.0
-Werkzeug>=2.1.2
-WTForms>=3.0.1
-xlrd>=2.0.1
+mdit-py-plugins==0.4.1
+mdurl==0.1.2
+myst-parser==3.0.1
+packaging==24.1
+Pygments==2.18.0
+PyYAML==6.0.1
+requests==2.32.3
+snowballstemmer==2.2.0
+Sphinx==7.4.7
+sphinx-rtd-theme==2.0.0
+sphinxcontrib-applehelp==1.0.8
+sphinxcontrib-devhelp==1.0.6
+sphinxcontrib-htmlhelp==2.0.6
+sphinxcontrib-jquery==4.1
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.8
+sphinxcontrib-serializinghtml==1.1.10
+tomli==2.0.1
+urllib3==2.2.2

hedweb/event_operations.py

@@ -123,17 +123,19 @@ def generate_sidecar(self):
                                                                  extension='.txt', append_datetime=True),
                         bc.MSG_CATEGORY: 'warning',
                         bc.MSG: f"Cannot generate sidecar because skipped and value column names overlap."}
-
-        columns_info = TabularSummary.get_columns_info(self.events.dataframe)
-        hed_dict = {}
-        for column_name in columns_info:
-            if column_name in self.columns_skip:
-                continue
-            elif column_name in self.columns_value:
-                hed_dict[column_name] = generate_sidecar_entry(column_name)
-            else:
-                hed_dict[column_name] = generate_sidecar_entry(column_name,
-                                                               column_values=list(columns_info[column_name].keys()))
+        tab_sum = TabularSummary(value_cols=self.columns_value, skip_cols=self.columns_skip)
+        tab_sum.update(self.events.dataframe)
+        hed_dict = tab_sum.extract_sidecar_template()
+        # columns_info = TabularSummary.get_columns_info(self.events.dataframe)
+        # hed_dict = {}
+        # for column_name in columns_info:
+        #     if column_name in self.columns_skip:
+        #         continue
+        #     elif column_name in self.columns_value:
+        #         hed_dict[column_name] = generate_sidecar_entry(column_name)
+        #     else:
+        #         hed_dict[column_name] = generate_sidecar_entry(column_name,
+        #                                                        column_values=list(columns_info[column_name].keys()))
         file_name = generate_filename(display_name, name_suffix='_generated', extension='.json', append_datetime=True)
         return {bc.COMMAND: bc.COMMAND_GENERATE_SIDECAR,
                 bc.COMMAND_TARGET: 'events',

hedweb/process_service.py

@@ -57,6 +57,7 @@ def set_parameters(arguments, params):
         arguments[bc.REQUEST_TYPE] = bc.FROM_SERVICE
         arguments[bc.COLUMNS_CATEGORICAL] = ProcessServices.get_list(bc.COLUMNS_CATEGORICAL, params)
         arguments[bc.COLUMNS_VALUE] = ProcessServices.get_list(bc.COLUMNS_VALUE, params)
+        arguments[bc.COLUMNS_SKIP] = ProcessServices.get_list(bc.COLUMNS_SKIP, params)
         arguments[bc.TAG_COLUMNS] = ProcessServices.get_list(bc.TAG_COLUMNS, params)
         arguments[bc.HAS_COLUMN_NAMES] = True
 

hedweb/templates/hed-tools-home.html

@@ -18,13 +18,14 @@ <h4>HED online tools summary</h4>
     The following table summarizes the available online tools. The tools are organized around the
         types of data that the tools handle. The Actions refer to the types of actions that can be performed on the
     data through the online tools. Each action is linked to its corresponding documentation to provide additional
-        information.</p>
+        information. <strong>NEW: an experimental <a href="http://ctagger.hed.tools">CTagger</a>
+        online annotation tool is now available</strong>.</p>
 
 
 
     <table class="table table-striped table-hover">
     <thead>
-       <tr><th scope="col">Data type<br/>(linked to form></th><th scope="col">Action<br/>(linked to docs)</th><th scope="col">Description</th>
+       <tr><th scope="col">Data type<br/>(form link)</th><th scope="col">Action<br/>(docs link)</th><th scope="col">Description</th>
     </thead>
         <tr class="table-active">
             <td><a href="{{ url_for('route_blueprint.render_events_form') }}"><strong>Events</strong></a></td>
@@ -112,17 +113,17 @@ <h4>HED online tools summary</h4>
         are used by schema developers creating the HED vocabularies and are not of interest to annotators or analysts.</p>
 
     <p>    
-    More detailed help on using these online tools is available at:
-        <a href="https://www.hed-resources.org/en/latest/HedOnlineTools.html#"><b>HED online tools</b></a>
+    More detailed help on using these online tools is available in the
+        <a href="https://www.hed-resources.org/en/latest/HedOnlineTools.html#"><b>HED online tools</b></a> documentation.
     </p>
 
    <h4>HED REST services</h4>
     <p>
         The HED online tools are also available as callable web services. More detailed help on calling these
         services is available at:
         <a href="https://www.hed-resources.org/en/latest/HedOnlineTools.html#hed-restful-services"><b>HED RESTful services</b></a>
-        Downloadable examples of calling these services from a MATLAB program can be found at:
-        <a href="https://github.com/hed-standard/hed-matlab/tree/main/hedmat/web_services">
+        Downloadable examples of calling these services from a MATLAB program can be found in the
+        <a href="https://github.com/hed-standard/hed-matlab/tree/main/hedmat/web_services_demos">
             GitHub MATLAB web service examples</a>.
      </p>
 

hedweb/templates/layout.html

@@ -63,6 +63,7 @@
              aria-haspopup="true" aria-expanded="false">Quick links</a>
           <div class="dropdown-menu">
             <a class="dropdown-item" href="https://www.hedtags.org/display_hed.html">HED schema browser</a>
+            <a class="dropdown-item" href="http://ctagger.hed.tools/">CTagger interactive annotation</a>
             <a class="dropdown-item" href="https://www.hedtags.org">HED homepage</a>
             <a class="dropdown-item" href="https://github.com/hed-standard">HED on GitHub</a>
             <a class="dropdown-item" href="https://www.youtube.com/playlist?list=PLeII6cRFsP6L5S6icwRrJp0DHkhOHtbp-">HED on YouTube</a>

hedweb/web_util.py

@@ -18,12 +18,11 @@
 
 def convert_hed_versions(hed_info):
     hed_list = []
+    standard_list = hed_info['schema_version_list'].get(None, [])
     for key, key_list in hed_info['schema_version_list'].items():
-        if key is None:
-            hed_list = hed_list + key_list
-        else:
+        if key is not None:
             hed_list = hed_list + [key + '_' + element for element in key_list]
-    return {'schema_version_list': hed_list}
+    return {'schema_version_list': standard_list + hed_list}
 
 
 def file_extension_is_valid(filename, accepted_extensions=None):

pyproject.toml

@@ -2,6 +2,50 @@
 requires = ["setuptools>=42", "versioneer[toml]"]
 build-backend = "setuptools.build_meta"
 
+[project]
+name = "hedweb"
+dynamic = ["version"]
+description = "Web interface, REST services, and Docker deployment for HED tools."
+readme = "README.md"
+authors = [
+    { name = "VisLab" },
+    { name = "Ian Callanan" },
+    { name = "Jeremy Cockfield" },
+    { name = "Kay Robbins", email = "[email protected]" }
+]
+license = { text = "MIT License" }
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent"
+]
+
+dependencies = [
+    "Flask",
+    "Flask-WTF",
+    "openpyxl",
+    "pandas",
+    "Werkzeug",
+    "requests"
+]
+
+[project.urls]
+"Homepage" = "https://github.com/hed-standard/hed-web/"
+"Bug Tracker" = "https://github.com/hed-standard/hed-web/issues"
+
+[tool.setuptools.packages.find]
+namespaces = true
+include = ["hed*"]
+
+[tool.versioneer]
+VCS = "git"
+style = "pep440"
+versionfile_source = "hedweb/_version.py"
+versionfile_build = "hedweb/_version.py"
+tag_prefix = ""
+parentdir_prefix = "hedweb-"
+
 [tool.codespell]
 skip = '*.git,*.pdf,*.xml,*.mediawiki,*.svg,versioneer.py,venv*,*.tsv,*.yaml,*.yml,*.json,*.rdf,*.jsonld,spec_tests,*.css,*.css.map'
-ignore-words-list = 'te,parms,cACE'
+ignore-words-list = 'te,parms,cACE,assertIn'

readthedocs.yml

@@ -7,7 +7,7 @@ formats:
 build:
   os: ubuntu-22.04
   tools:
-    python: '3.8'
+    python: '3.10'
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

requirements.txt

@@ -1,6 +1,5 @@
-flask>=3.0.3
-flask-wtf>=1.2.1
-config>=0.5.1
+Flask>=3.0.3
+Flask-WTF>=1.2.1
 openpyxl>=3.1.2
 pandas>=2.2.0
 Werkzeug>=3.0.1

tests/test_events.py

@@ -1,5 +1,6 @@
 import os
 from io import StringIO
+import json
 import pandas as pd
 import unittest
 from werkzeug.test import create_environ
@@ -167,6 +168,10 @@ def test_generate_sidecar_valid(self):
                         'generate_sidecar results should have a data key when no errors')
         self.assertEqual('success', results['msg_category'],
                          'generate_sidecar msg_category should be success when no errors')
+        sidecar_template = json.loads(results['data'])
+        self.assertFalse('onset' in sidecar_template)
+        self.assertIsInstance(sidecar_template['event_type']['HED'], dict)
+        self.assertTrue(isinstance(sidecar_template['trial']['HED'], str))
 
     def test_search_invalid(self):
         with self.app.app_context():

tests/test_web_util.py

@@ -9,6 +9,21 @@
 
 class Test(TestWebBase):
 
+    def test_convert_hed_versions(self):
+        from hedweb.web_util import convert_hed_versions
+        no_none = {'schema_version_list': {'score': ['1.0.0', '2.0.0']}}
+        no_none_list = convert_hed_versions(no_none)
+        self.assertTrue(no_none_list['schema_version_list'] == ['score_1.0.0', 'score_2.0.0'])
+        just_none = {'schema_version_list': {None: ['8.0.0', '8.1.0']}}
+        just_none_list = convert_hed_versions(just_none)
+        self.assertTrue(just_none_list['schema_version_list'] == ['8.0.0', '8.1.0'])
+        just_blank = {'schema_version_list': {'': ['9.0.0', '9.1.0']}}
+        just_blank_list = convert_hed_versions(just_blank)
+        self.assertTrue(just_blank_list['schema_version_list'] == ['_9.0.0', '_9.1.0'])
+        test_all = {'schema_version_list': {'score': ['1.0.0', '2.0.0'], None: ['8.0.0', '8.1.0']}}
+        test_all_list = convert_hed_versions(test_all)
+        self.assertTrue(test_all_list['schema_version_list'] == ['8.0.0', '8.1.0', 'score_1.0.0', 'score_2.0.0'])
+
     def test_form_has_file(self):
         from hedweb.web_util import form_has_file
         with self.app.test as _:
@@ -241,7 +256,6 @@ def test_get_hed_schema_from_pull_down_empty(self):
 
     def test_get_hed_schema_from_pull_down_version(self):
         from hed.schema import HedSchema
-        from hedweb.constants import base_constants as bc
         from hedweb.web_util import get_hed_schema_from_pull_down
         with self.app.test:
             environ = create_environ(data={bc.SCHEMA_VERSION: '8.0.0'})
@@ -252,7 +266,6 @@ def test_get_hed_schema_from_pull_down_version(self):
 
     def test_get_hed_schema_from_pull_down_other(self):
         from hed.schema import HedSchema
-        from hedweb.constants import base_constants as bc
         from hedweb.web_util import get_hed_schema_from_pull_down
         with self.app.test:
             schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/HED8.0.0.xml')