Merge pull request #189 from VisLab/develop

Changed schema encoding to utf-8

deploy_hed/requirements.txt

@@ -1,8 +1,8 @@
-flask>=3.0.2
+flask>=3.0.3
 flask-wtf>=1.2.1
 config>=0.5.1
 mod_wsgi>=4.9.0
 openpyxl>=3.1.2
 pandas>=2.2.0
-Werkzeug>=3.0.1
+Werkzeug>=3.0.3
 requests>=2.31.0

deploy_hed_dev/requirements.txt

@@ -1,4 +1,4 @@
-flask>=3.0.2
+flask>=3.0.3
 flask-wtf>=1.2.1
 config>=0.5.1
 mod_wsgi>=4.9.0

docs/requirements.txt

@@ -1,29 +1,29 @@
 attrs>=21.4.0
-click==8.1.3
+click>=8.1.3
 coverage>=6.3.2
-defusedxml==0.7.1
-et-xmlfile==1.1.0
-Flask==2.1.2
-Flask-WTF==1.0.1
+defusedxml>=0.7.1
+et-xmlfile>=1.1.0
+Flask>=3.0.3
+Flask-WTF>=1.2.1
 inflect>=5.5.2
-itsdangerous==2.1.2
-jdcal==1.4.1
+itsdangerous>=2.2.0
+jdcal>=1.4.1
 Jinja2>=3.1.2
 MarkupSafe==2.1.1
 myst-parser>=0.17.0
 numpy>=1.20.3
-numpydoc==1.3.1
+numpydoc>=1.3.1
 openpyxl>=3.0.9
 pandas>=1.3.5
-portalocker==2.4.0
-Pygments==2.12.0
-python-dateutil==2.8.2
-pytz==2022.1
+portalocker>=2.8.2
+Pygments>=2.12.0
+python-dateutil>=2.8.2
+pytz>=2022.1
 semantic_version>=2.9.0
 six==1.16.0
 Sphinx>=4,<5
-SphinxExtensions==0.2.0
-sphinx_rtd_theme==1.0.0
-Werkzeug==2.1.2
-WTForms==3.0.1
-xlrd==2.0.1
+SphinxExtensions>=0.2.0
+sphinx_rtd_theme>=1.0.0
+Werkzeug>=2.1.2
+WTForms>=3.0.1
+xlrd>=2.0.1

hedweb/process_form.py

@@ -117,7 +117,7 @@ def set_schema_from_request(arguments, request):
         elif form_has_option(request.form, bc.SCHEMA_VERSION) and form_has_file(request.files, bc.SCHEMA_PATH):
             f = request.files[bc.SCHEMA_PATH]
             arguments[bc.SCHEMA] = \
-                from_string(f.read(fc.BYTE_LIMIT).decode('ascii'), schema_format=secure_filename(f.filename))
+                from_string(f.read(fc.BYTE_LIMIT).decode('utf-8'), schema_format=secure_filename(f.filename))
         if form_has_option(request.form, bc.SCHEMA_UPLOAD_OPTIONS, bc.SCHEMA_FILE_OPTION) and \
                 form_has_file(request.files, bc.SCHEMA_FILE, fc.SCHEMA_EXTENSIONS):
             arguments[bc.SCHEMA] = ProcessForm.get_schema(request.files[bc.SCHEMA_FILE])
@@ -148,7 +148,7 @@ def get_schema(schema_input=None, version=None, as_xml_string=None):
         """
         if isinstance(schema_input, FileStorage):
             name, extension = get_parsed_name(secure_filename(schema_input.filename))
-            hed_schema = hedschema.from_string(schema_input.read(fc.BYTE_LIMIT).decode('ascii'),
+            hed_schema = hedschema.from_string(schema_input.read(fc.BYTE_LIMIT).decode('utf-8'),
                                                schema_format=extension,
                                                name=name)
         elif isinstance(schema_input, str):

hedweb/routes.py

@@ -101,7 +101,7 @@ def schema_version_results():
         if bc.SCHEMA_PATH in request.files:
             f = request.files[bc.SCHEMA_PATH]
             name, extension = get_parsed_name(secure_filename(f.filename))
-            hed_schema = hedschema.from_string(f.stream.read(file_constants.BYTE_LIMIT).decode('ascii'),
+            hed_schema = hedschema.from_string(f.stream.read(file_constants.BYTE_LIMIT).decode('utf-8'),
                                                schema_format=extension)
             hed_info[bc.SCHEMA_VERSION] = hed_schema.get_formatted_version()
         return json.dumps(hed_info)

hedweb/schema_operations.py

@@ -148,7 +148,7 @@ def get_schema(schema_input=None, version=None, as_xml_string=None):
     """
     if isinstance(schema_input, FileStorage):
         name, extension = get_parsed_name(secure_filename(schema_input.filename))
-        hed_schema = hedschema.from_string(schema_input.read(fc.BYTE_LIMIT).decode('ascii'),
+        hed_schema = hedschema.from_string(schema_input.read(fc.BYTE_LIMIT).decode('utf-8'),
                                            schema_format=extension,
                                            name=name)
     elif isinstance(schema_input, str):

hedweb/web_util.py

@@ -270,7 +270,7 @@ def get_hed_schema_from_pull_down(request):
         hed_schema = load_schema_version(request.form[bc.SCHEMA_VERSION])
     elif bc.SCHEMA_PATH in request.files:
         f = request.files[bc.SCHEMA_PATH]
-        hed_schema = hedschema.from_string(f.read(fc.BYTE_LIMIT).decode('ascii'),
+        hed_schema = hedschema.from_string(f.read(fc.BYTE_LIMIT).decode('utf-8'),
                                            schema_format=secure_filename(f.filename))
     else:
         raise HedFileError("NoSchemaFile", "Must provide a valid schema for upload if other chosen", "")

requirements.txt

@@ -1,4 +1,4 @@
-flask>=3.0.2
+flask>=3.0.3
 flask-wtf>=1.2.1
 config>=0.5.1
 openpyxl>=3.1.2

services_tests/test_services_base.py

@@ -80,7 +80,7 @@ def _get_path(self, filename):
     def _get_file_string(self, filename):
         filename = self._get_path(filename)
         with open(filename, 'rb') as fp:
-            filename_string = fp.read().decode('ascii')
+            filename_string = fp.read().decode('utf-8')
 
         return filename_string
 

tests/test_routes/test_routes_base.py

@@ -28,6 +28,6 @@ def _get_file_buffer(self, filename, name=None):
     def _get_file_string(self, filename):
         filename = self._get_path(filename)
         with open(filename, 'rb') as fp:
-            filename_string = fp.read().decode('ascii')
+            filename_string = fp.read().decode('utf-8')
 
         return filename_string

tests/test_services.py

@@ -45,7 +45,7 @@ def test_set_input_from_service_request(self):
         with self.app.test:
             sidecar_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/bids_events.json')
             with open(sidecar_path, 'rb') as fp:
-                sidecar_string = fp.read().decode('ascii')
+                sidecar_string = fp.read().decode('utf-8')
             json_data = {bc.SIDECAR_STRING: sidecar_string, bc.CHECK_FOR_WARNINGS: 'on',
                          bc.SCHEMA_VERSION: '8.2.0', bc.SERVICE: 'sidecar_validate'}
             environ = create_environ(json=json_data)
@@ -61,7 +61,7 @@ def test_set_input_from_service_request_full_template(self):
         with self.app.test:
             sidecar_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/bids_events.json')
             with open(sidecar_path, 'rb') as fp:
-                sidecar_string = fp.read().decode('ascii')
+                sidecar_string = fp.read().decode('utf-8')
             json_data = self.get_request_template()
             json_data[bc.SIDECAR_STRING] = sidecar_string
             json_data[bc.CHECK_FOR_WARNINGS] = True,

tests/test_web_util.py

@@ -220,7 +220,7 @@ def test_generate_text_response(self):
                              "generate_download_download_text_response should return text")
             self.assertEqual(results[bc.MSG], headers_dict['Message'],
                              "generate_download_text_response have the correct message in the response")
-            self.assertEqual(results['data'], response.data.decode('ascii'),
+            self.assertEqual(results['data'], response.data.decode('utf-8'),
                              "generate_download_text_response have the download text as response data")
 
     def test_get_hed_schema_from_pull_down_empty(self):