Merge branch 'main' into catalog-next

.github/workflows/snyk.yml

@@ -2,6 +2,9 @@
 name: Check for Snyk Vulnerabilities
 
 on:
+  pull_request:
+    branches:
+      - main
   workflow_dispatch:
   schedule:
     - cron: '0 12 * * *'  # every day at 12pm UTC
@@ -56,7 +59,7 @@ jobs:
           # Fail so that PR is created
           exit 1
       - name: Create Pull Request
-        if: ${{ failure() }}
+        if: ${{ failure() && github.event_name == 'schedule' }}
         id: scpr
         uses: peter-evans/create-pull-request@v5
         with:

ckan/.snyk

@@ -7,77 +7,59 @@ ignore:
         reason: >-
           No remediation available yet; Not affecting us since the storage is
           not accessible to any other client
-        expires: 2024-07-31T19:29:54.032Z
+        expires: 2024-11-30T19:29:54.032Z
         created: 2022-12-08T16:20:58.023Z
   SNYK-PYTHON-WERKZEUG-6035177:
     - '*':
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-07-31T19:29:54.032Z
+        expires: 2024-11-30T19:29:54.032Z
         created: 2023-10-30T16:50:58.023Z
   SNYK-PYTHON-WERKZEUG-3319936:
     - '*':
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-07-31T19:29:54.032Z
+        expires: 2024-11-30T19:29:54.032Z
         created: 2023-02-15T16:20:58.023Z
   SNYK-PYTHON-WERKZEUG-3319935:
     - '*':
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-07-31T19:29:54.032Z
+        expires: 2024-11-30T19:29:54.032Z
         created: 2023-02-15T16:20:58.023Z
   SNYK-PYTHON-FLASK-5490129:
     - '*':
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4303
-        expires: 2024-07-31T19:29:54.032Z
+        expires: 2024-11-30T19:29:54.032Z
         created: 2023-05-08T16:20:58.023Z
   SNYK-PYTHON-PYOPENSSL-6149520:
     - '*':
         reason: >-
           No remediation available yet; Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4532
-        expires: 2024-07-31T19:29:54.032Z
+        expires: 2024-11-30T19:29:54.032Z
         created: 2024-01-08T00:00:00.000Z
   SNYK-PYTHON-PYOPENSSL-6157250:
     - '*':
         reason: >-
           No remediation available yet; Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4591
-        expires: 2024-07-31T19:29:54.032Z
+        expires: 2024-11-30T19:29:54.032Z
         created: 2024-01-14T00:00:00.000Z
-  SNYK-PYTHON-CRYPTOGRAPHY-6592767:
-    - '*':
-        reason: >-
-          No remediation available yet; Low severity.
-        expires: 2024-10-24T17:21:30.083Z
-        created: 2024-04-24T17:21:30.089Z
   SNYK-PYTHON-PYOPENSSL-6592766:
     - '*':
         reason: >-
           No remediation available yet; Low severity.
-        expires: 2024-10-24T17:24:47.251Z
+        expires: 2024-11-30T17:24:47.251Z
         created: 2024-04-24T17:24:47.257Z
   SNYK-PYTHON-WERKZEUG-6808933:
     - '*':
         reason: >-
            Not affecting us since no debugger is enabled in cloud.gov apps
-        expires: 2024-06-31T16:20:58.017Z
-  SNYK-PYTHON-CRYPTOGRAPHY-7161587:
-    - '*':
-        reason: >-
-           No remediation available yet. Issue tracked in github:
-           https://github.com/GSA/data.gov/issues/4781
-        expires: 2024-06-31T16:20:58.017Z
-  SNYK-PYTHON-PYOPENSSL-7161590:
-    - '*':
-        reason: >-
-           No remediation available yet. Issue tracked in github:
-           https://github.com/GSA/data.gov/issues/4782
-        expires: 2024-06-31T16:20:58.017Z
+        expires: 2024-11-30T16:20:58.017Z
 patch: {}

ckan/requirements.in

@@ -1,8 +1,8 @@
 # CKAN requirements and extensions
-git+https://github.com/GSA/ckan.git@ckan-2-10-4-fork#egg=ckan
-git+https://github.com/ckan/ckanext-dcat@master#egg=ckanext-dcat
+git+https://github.com/GSA/ckan.git@ckan-2-10-5-fork#egg=ckan
+git+https://github.com/ckan/ckanext-dcat@v1.7.0#egg=ckanext-dcat
 -e git+https://github.com/GSA/ckanext-harvest.git@release-v1-5-6#egg=ckanext-harvest
--e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial
+-e git+https://github.com/GSA/ckanext-spatial.git@iis-dir#egg=ckanext-spatial
 git+https://github.com/GSA/ckanext-saml2auth.git@datagov#egg=ckanext-saml2auth
 # -e git+https://github.com/ckan/ckanext-qa.git@master#egg=ckanext-qa
 -e git+https://github.com/ckan/ckanext-archiver.git@master#egg=ckanext-archiver
@@ -77,11 +77,12 @@ Flask-WTF==1.0.1
 flask-multistatic==1.0
 greenlet==2.0.2
 #Jinja2==3.1.2
-PyJWT==2.4.0
 Markdown==3.4.1
+packaging==24.1
 passlib==1.7.4
 polib==1.1.1
 psycopg2==2.9.3
+PyJWT==2.4.0
 python-magic==0.4.27
 pysolr==3.9.0
 python-dateutil==2.8.2
@@ -112,14 +113,13 @@ gunicorn
 
 # New Relic
 newrelic
-certifi>=2022.12.7
 redis>=4.5.4
-requests~=2.32.2
+requests~=2.32.3
 
 # avoid ImportError error https://github.com/GSA/data.gov/issues/4396
 importlib-resources<6.0
 gevent>=23.9.0
-jinja2>=3.1.3
+jinja2>=3.1.4
 cryptography>=42.0.4
 
 # lxml beyond 5.1.0 show error module 'lxml.etree' has no attribute '_ElementStringResult'
@@ -131,4 +131,8 @@ lxml==5.1.0
 Werkzeug==2.0.3
 
 # pin numpy as 2.x causes array import issues w/ shapely
-numpy==1.26.4
+numpy==1.26.4
+certifi>=2024.7.4
+
+# snyk finding
+setuptools~=71.0.3

ckan/requirements.txt

@@ -4,28 +4,28 @@ Babel==2.10.3
 Beaker==1.11.0
 bleach==5.0.1
 blinker==1.5
-boto3==1.34.128
-botocore==1.34.128
-certifi==2024.6.2
-cffi==1.16.0
+boto3==1.35.12
+botocore==1.35.12
+certifi==2024.8.30
+cffi==1.17.0
 chardet==5.2.0
 charset-normalizer==3.3.2
-ckan @ git+https://github.com/GSA/ckan.git@7159a872ba740069b768fcd2a43cde81a57ee492
+ckan @ git+https://github.com/GSA/ckan.git@8c4a517efeac80db098cc6ba144cb742bbeca194
 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver
 -e git+https://github.com/GSA/ckanext-datagovcatalog.git@harvest-next#egg=ckanext_datagovcatalog
 -e git+https://github.com/GSA/ckanext-datagovtheme.git@harvest-next#egg=ckanext_datagovtheme
 ckanext-datajson==0.1.25
-ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@83495ba99cba17398ba8feb1bc0da486f3798584
+ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@b8ebf24004cd3f3edb7f9d01c87c20259c102093
 ckanext-envvars==0.0.3
 ckanext-geodatagov==0.2.9
 -e git+https://github.com/GSA/ckanext-harvest.git@9039e7a5d563a40177d62487758b366ab77434b6#egg=ckanext_harvest
 ckanext-metrics-dashboard==0.1.6
 -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report
 ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@387cfc1c6a7619f670bf387384f2634516de5844
--e git+https://github.com/ckan/ckanext-spatial.git@938308469892e4bcf7389cb4adee5ccdd5a0ccca#egg=ckanext_spatial
+-e git+https://github.com/GSA/ckanext-spatial.git@3d0a375fe98edc70a0d12efd2f4ac54f0e05b597#egg=ckanext_spatial
 ckantoolkit==0.0.7
 click==8.1.3
-cryptography==42.0.8
+cryptography==43.0.1
 defusedxml==0.7.1
 dominate==2.7.0
 elementpath==4.4.0
@@ -41,9 +41,9 @@ geojson==3.0.1
 geomet==1.1.0
 gevent==24.2.1
 greenlet==2.0.2
-gunicorn==22.0.0
+gunicorn==23.0.0
 html5lib==1.1
-idna==3.7
+idna==3.8
 importlib-resources==5.13.0
 isodate==0.6.1
 itsdangerous==2.2.0
@@ -56,25 +56,25 @@ Mako==1.3.5
 Markdown==3.4.1
 MarkupSafe==2.1.5
 messytables==0.15.2
-mypy==1.10.0
+mypy==1.10.1
 mypy-extensions==1.0.0
-newrelic==9.11.0
+newrelic==9.13.0
 nose==1.3.7
 numpy==1.26.4
 OWSLib==0.31.0
 packaging==24.1
 passlib==1.7.4
 pika==1.2.1
-pip==24.0
+pip==24.1
 ply==3.11
 polib==1.1.1
 progressbar==2.5
 progressbar2==3.53.3
 psycopg2==2.9.3
 pycparser==2.22
 PyJWT==2.4.0
-pyOpenSSL==24.1.0
-pyparsing==3.1.2
+pyOpenSSL==24.2.1
+pyparsing==3.1.4
 pyproj==3.4.1
 pysaml2==7.0.1
 pysolr==3.9.0
@@ -87,13 +87,13 @@ PyUtilib==6.0.0
 PyYAML==6.0.1
 PyZ3950 @ git+https://github.com/danizen/PyZ3950@6d44a4ab85c8bda3a7542c2c9efdfad46c830219
 rdflib==6.1.1
-redis==5.0.6
+redis==5.0.8
 requests==2.32.3
 rfc3987==1.3.8
 rq==1.11.0
-s3transfer==0.10.1
+s3transfer==0.10.2
 sansjson==0.3.0
-setuptools==67.1.0
+setuptools==71.0.4
 shapely==2.0.1
 simplejson==3.18.0
 six==1.16.0
@@ -105,13 +105,13 @@ typing_extensions==4.3.0
 tzdata==2024.1
 tzlocal==4.2
 urllib3==2.2.2
-watchdog==4.0.1
+watchdog==5.0.2
 webassets==2.0
 webencodings==0.5.1
 Werkzeug==2.0.3
 wheel==0.42.0
 WTForms==3.1.2
 xlrd==2.0.1
-xmlschema==3.3.1
+xmlschema==3.3.2
 zope.event==5.0
 zope.interface==5.4.0

ckan/setup/ckan.ini

@@ -42,6 +42,8 @@ beaker.session.secret = TShFJxS41xNdVJAxQsoIEm5zu
 beaker.session.type=ext:database
 #beaker.session.url=postgresql://ckan:ckan@db/ckan
 beaker.session.cookie_expires=true
+beaker.session.secure = True
+beaker.session.samesite = Lax
 beaker.session.url = $CKAN___BEAKER__SESSION__URL
 beaker.session.timeout=900
 

e2e/cypress/integration/ckan_extensions.cy.js

@@ -2,7 +2,7 @@ describe('CKAN Extensions', () => {
     it('Uses CKAN 2.10', () => {
         cy.request('/api/action/status_show').should((response) => {
             expect(response.body).to.have.property('success', true);
-            expect(response.body.result).to.have.property('ckan_version', '2.10.4');
+            expect(response.body.result).to.have.property('ckan_version', '2.10.5');
         });
     });
 

proxy/public/500.html

@@ -3,7 +3,7 @@
 <!--[if IE 9]> <html lang="en" class="ie9"> <![endif]-->
 <!--[if gt IE 8]><!--> <html lang="en"  > <!--<![endif]-->
   <head>
-      <meta name="generator" content="ckan 2.10.4" />
+      <meta name="generator" content="ckan 2.10.5" />
       <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>500 Web server unavailable</title>
 

proxy/public/maintenance.html

@@ -3,7 +3,7 @@
 <!--[if IE 9]> <html lang="en" class="ie9"> <![endif]-->
 <!--[if gt IE 8]><!--> <html lang="en"  > <!--<![endif]-->
   <head>
-      <meta name="generator" content="ckan 2.10.4" />
+      <meta name="generator" content="ckan 2.10.5" />
       <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>503 Site under maintenance</title>
 

proxy/public/sitedown.html

@@ -3,7 +3,7 @@
 <!--[if IE 9]> <html lang="en" class="ie9"> <![endif]-->
 <!--[if gt IE 8]><!--> <html lang="en"  > <!--<![endif]-->
   <head>
-      <meta name="generator" content="ckan 2.10.4" />
+      <meta name="generator" content="ckan 2.10.5" />
       <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>503 Site Temporarily Down</title>
 

proxy/public/template.html

@@ -3,7 +3,7 @@
 <!--[if IE 9]> <html lang="en" class="ie9"> <![endif]-->
 <!--[if gt IE 8]><!--> <html lang="en"  > <!--<![endif]-->
   <head>
-      <meta name="generator" content="ckan 2.10.4" />
+      <meta name="generator" content="ckan 2.10.5" />
       <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>Error 404 - Catalog</title>
 

tools/harvest_source_import/dev-requirements.txt

@@ -0,0 +1,5 @@
+-r requirements.txt
+pytest>=5.4.2
+pytest-vcr>=1.0.2
+flake8>=3.8.1
+zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability

tools/harvest_source_import/requirements.txt

@@ -0,0 +1,4 @@
+requests>=2.32.0
+pytest>=5.4.2
+pytest-vcr>=1.0.2
+zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability