diff --git a/.github/workflows/snyk.yml b/.github/workflows/snyk.yml index 3b9ec11e..74c74688 100644 --- a/.github/workflows/snyk.yml +++ b/.github/workflows/snyk.yml @@ -2,6 +2,9 @@ name: Check for Snyk Vulnerabilities on: + pull_request: + branches: + - main workflow_dispatch: schedule: - cron: '0 12 * * *' # every day at 12pm UTC @@ -56,7 +59,7 @@ jobs: # Fail so that PR is created exit 1 - name: Create Pull Request - if: ${{ failure() }} + if: ${{ failure() && github.event_name == 'schedule' }} id: scpr uses: peter-evans/create-pull-request@v5 with: diff --git a/ckan/.snyk b/ckan/.snyk index 58781bdd..83015824 100644 --- a/ckan/.snyk +++ b/ckan/.snyk @@ -7,77 +7,59 @@ ignore: reason: >- No remediation available yet; Not affecting us since the storage is not accessible to any other client - expires: 2024-07-31T19:29:54.032Z + expires: 2024-11-30T19:29:54.032Z created: 2022-12-08T16:20:58.023Z SNYK-PYTHON-WERKZEUG-6035177: - '*': reason: >- Upgrade path is complex, Issue tracked in github: https://github.com/GSA/data.gov/issues/4217 - expires: 2024-07-31T19:29:54.032Z + expires: 2024-11-30T19:29:54.032Z created: 2023-10-30T16:50:58.023Z SNYK-PYTHON-WERKZEUG-3319936: - '*': reason: >- Upgrade path is complex, Issue tracked in github: https://github.com/GSA/data.gov/issues/4217 - expires: 2024-07-31T19:29:54.032Z + expires: 2024-11-30T19:29:54.032Z created: 2023-02-15T16:20:58.023Z SNYK-PYTHON-WERKZEUG-3319935: - '*': reason: >- Upgrade path is complex, Issue tracked in github: https://github.com/GSA/data.gov/issues/4217 - expires: 2024-07-31T19:29:54.032Z + expires: 2024-11-30T19:29:54.032Z created: 2023-02-15T16:20:58.023Z SNYK-PYTHON-FLASK-5490129: - '*': reason: >- Upgrade path is complex, Issue tracked in github: https://github.com/GSA/data.gov/issues/4303 - expires: 2024-07-31T19:29:54.032Z + expires: 2024-11-30T19:29:54.032Z created: 2023-05-08T16:20:58.023Z SNYK-PYTHON-PYOPENSSL-6149520: - '*': reason: >- No remediation available yet; Issue tracked in github: https://github.com/GSA/data.gov/issues/4532 - expires: 2024-07-31T19:29:54.032Z + expires: 2024-11-30T19:29:54.032Z created: 2024-01-08T00:00:00.000Z SNYK-PYTHON-PYOPENSSL-6157250: - '*': reason: >- No remediation available yet; Issue tracked in github: https://github.com/GSA/data.gov/issues/4591 - expires: 2024-07-31T19:29:54.032Z + expires: 2024-11-30T19:29:54.032Z created: 2024-01-14T00:00:00.000Z - SNYK-PYTHON-CRYPTOGRAPHY-6592767: - - '*': - reason: >- - No remediation available yet; Low severity. - expires: 2024-10-24T17:21:30.083Z - created: 2024-04-24T17:21:30.089Z SNYK-PYTHON-PYOPENSSL-6592766: - '*': reason: >- No remediation available yet; Low severity. - expires: 2024-10-24T17:24:47.251Z + expires: 2024-11-30T17:24:47.251Z created: 2024-04-24T17:24:47.257Z SNYK-PYTHON-WERKZEUG-6808933: - '*': reason: >- Not affecting us since no debugger is enabled in cloud.gov apps - expires: 2024-06-31T16:20:58.017Z - SNYK-PYTHON-CRYPTOGRAPHY-7161587: - - '*': - reason: >- - No remediation available yet. Issue tracked in github: - https://github.com/GSA/data.gov/issues/4781 - expires: 2024-06-31T16:20:58.017Z - SNYK-PYTHON-PYOPENSSL-7161590: - - '*': - reason: >- - No remediation available yet. Issue tracked in github: - https://github.com/GSA/data.gov/issues/4782 - expires: 2024-06-31T16:20:58.017Z + expires: 2024-11-30T16:20:58.017Z patch: {} diff --git a/ckan/requirements.in b/ckan/requirements.in index 7f96c7df..9d8ea8dc 100644 --- a/ckan/requirements.in +++ b/ckan/requirements.in @@ -1,8 +1,8 @@ # CKAN requirements and extensions -git+https://github.com/GSA/ckan.git@ckan-2-10-4-fork#egg=ckan -git+https://github.com/ckan/ckanext-dcat@master#egg=ckanext-dcat +git+https://github.com/GSA/ckan.git@ckan-2-10-5-fork#egg=ckan +git+https://github.com/ckan/ckanext-dcat@v1.7.0#egg=ckanext-dcat -e git+https://github.com/GSA/ckanext-harvest.git@release-v1-5-6#egg=ckanext-harvest --e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial +-e git+https://github.com/GSA/ckanext-spatial.git@iis-dir#egg=ckanext-spatial git+https://github.com/GSA/ckanext-saml2auth.git@datagov#egg=ckanext-saml2auth # -e git+https://github.com/ckan/ckanext-qa.git@master#egg=ckanext-qa -e git+https://github.com/ckan/ckanext-archiver.git@master#egg=ckanext-archiver @@ -77,11 +77,12 @@ Flask-WTF==1.0.1 flask-multistatic==1.0 greenlet==2.0.2 #Jinja2==3.1.2 -PyJWT==2.4.0 Markdown==3.4.1 +packaging==24.1 passlib==1.7.4 polib==1.1.1 psycopg2==2.9.3 +PyJWT==2.4.0 python-magic==0.4.27 pysolr==3.9.0 python-dateutil==2.8.2 @@ -112,14 +113,13 @@ gunicorn # New Relic newrelic -certifi>=2022.12.7 redis>=4.5.4 -requests~=2.32.2 +requests~=2.32.3 # avoid ImportError error https://github.com/GSA/data.gov/issues/4396 importlib-resources<6.0 gevent>=23.9.0 -jinja2>=3.1.3 +jinja2>=3.1.4 cryptography>=42.0.4 # lxml beyond 5.1.0 show error module 'lxml.etree' has no attribute '_ElementStringResult' @@ -131,4 +131,8 @@ lxml==5.1.0 Werkzeug==2.0.3 # pin numpy as 2.x causes array import issues w/ shapely -numpy==1.26.4 \ No newline at end of file +numpy==1.26.4 +certifi>=2024.7.4 + +# snyk finding +setuptools~=71.0.3 diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 26760463..f81ac62d 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -4,28 +4,28 @@ Babel==2.10.3 Beaker==1.11.0 bleach==5.0.1 blinker==1.5 -boto3==1.34.128 -botocore==1.34.128 -certifi==2024.6.2 -cffi==1.16.0 +boto3==1.35.12 +botocore==1.35.12 +certifi==2024.8.30 +cffi==1.17.0 chardet==5.2.0 charset-normalizer==3.3.2 -ckan @ git+https://github.com/GSA/ckan.git@7159a872ba740069b768fcd2a43cde81a57ee492 +ckan @ git+https://github.com/GSA/ckan.git@8c4a517efeac80db098cc6ba144cb742bbeca194 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/GSA/ckanext-datagovcatalog.git@harvest-next#egg=ckanext_datagovcatalog -e git+https://github.com/GSA/ckanext-datagovtheme.git@harvest-next#egg=ckanext_datagovtheme ckanext-datajson==0.1.25 -ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@83495ba99cba17398ba8feb1bc0da486f3798584 +ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@b8ebf24004cd3f3edb7f9d01c87c20259c102093 ckanext-envvars==0.0.3 ckanext-geodatagov==0.2.9 -e git+https://github.com/GSA/ckanext-harvest.git@9039e7a5d563a40177d62487758b366ab77434b6#egg=ckanext_harvest ckanext-metrics-dashboard==0.1.6 -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@387cfc1c6a7619f670bf387384f2634516de5844 --e git+https://github.com/ckan/ckanext-spatial.git@938308469892e4bcf7389cb4adee5ccdd5a0ccca#egg=ckanext_spatial +-e git+https://github.com/GSA/ckanext-spatial.git@3d0a375fe98edc70a0d12efd2f4ac54f0e05b597#egg=ckanext_spatial ckantoolkit==0.0.7 click==8.1.3 -cryptography==42.0.8 +cryptography==43.0.1 defusedxml==0.7.1 dominate==2.7.0 elementpath==4.4.0 @@ -41,9 +41,9 @@ geojson==3.0.1 geomet==1.1.0 gevent==24.2.1 greenlet==2.0.2 -gunicorn==22.0.0 +gunicorn==23.0.0 html5lib==1.1 -idna==3.7 +idna==3.8 importlib-resources==5.13.0 isodate==0.6.1 itsdangerous==2.2.0 @@ -56,16 +56,16 @@ Mako==1.3.5 Markdown==3.4.1 MarkupSafe==2.1.5 messytables==0.15.2 -mypy==1.10.0 +mypy==1.10.1 mypy-extensions==1.0.0 -newrelic==9.11.0 +newrelic==9.13.0 nose==1.3.7 numpy==1.26.4 OWSLib==0.31.0 packaging==24.1 passlib==1.7.4 pika==1.2.1 -pip==24.0 +pip==24.1 ply==3.11 polib==1.1.1 progressbar==2.5 @@ -73,8 +73,8 @@ progressbar2==3.53.3 psycopg2==2.9.3 pycparser==2.22 PyJWT==2.4.0 -pyOpenSSL==24.1.0 -pyparsing==3.1.2 +pyOpenSSL==24.2.1 +pyparsing==3.1.4 pyproj==3.4.1 pysaml2==7.0.1 pysolr==3.9.0 @@ -87,13 +87,13 @@ PyUtilib==6.0.0 PyYAML==6.0.1 PyZ3950 @ git+https://github.com/danizen/PyZ3950@6d44a4ab85c8bda3a7542c2c9efdfad46c830219 rdflib==6.1.1 -redis==5.0.6 +redis==5.0.8 requests==2.32.3 rfc3987==1.3.8 rq==1.11.0 -s3transfer==0.10.1 +s3transfer==0.10.2 sansjson==0.3.0 -setuptools==67.1.0 +setuptools==71.0.4 shapely==2.0.1 simplejson==3.18.0 six==1.16.0 @@ -105,13 +105,13 @@ typing_extensions==4.3.0 tzdata==2024.1 tzlocal==4.2 urllib3==2.2.2 -watchdog==4.0.1 +watchdog==5.0.2 webassets==2.0 webencodings==0.5.1 Werkzeug==2.0.3 wheel==0.42.0 WTForms==3.1.2 xlrd==2.0.1 -xmlschema==3.3.1 +xmlschema==3.3.2 zope.event==5.0 zope.interface==5.4.0 diff --git a/ckan/setup/ckan.ini b/ckan/setup/ckan.ini index dc51e209..b525b8a9 100644 --- a/ckan/setup/ckan.ini +++ b/ckan/setup/ckan.ini @@ -42,6 +42,8 @@ beaker.session.secret = TShFJxS41xNdVJAxQsoIEm5zu beaker.session.type=ext:database #beaker.session.url=postgresql://ckan:ckan@db/ckan beaker.session.cookie_expires=true +beaker.session.secure = True +beaker.session.samesite = Lax beaker.session.url = $CKAN___BEAKER__SESSION__URL beaker.session.timeout=900 diff --git a/e2e/cypress/integration/ckan_extensions.cy.js b/e2e/cypress/integration/ckan_extensions.cy.js index 97526a3c..33f7f73c 100644 --- a/e2e/cypress/integration/ckan_extensions.cy.js +++ b/e2e/cypress/integration/ckan_extensions.cy.js @@ -2,7 +2,7 @@ describe('CKAN Extensions', () => { it('Uses CKAN 2.10', () => { cy.request('/api/action/status_show').should((response) => { expect(response.body).to.have.property('success', true); - expect(response.body.result).to.have.property('ckan_version', '2.10.4'); + expect(response.body.result).to.have.property('ckan_version', '2.10.5'); }); }); diff --git a/proxy/public/500.html b/proxy/public/500.html index 32ac3c34..923d3353 100644 --- a/proxy/public/500.html +++ b/proxy/public/500.html @@ -3,7 +3,7 @@ - + 500 Web server unavailable diff --git a/proxy/public/maintenance.html b/proxy/public/maintenance.html index ac45b066..e56a0edd 100644 --- a/proxy/public/maintenance.html +++ b/proxy/public/maintenance.html @@ -3,7 +3,7 @@ - + 503 Site under maintenance diff --git a/proxy/public/sitedown.html b/proxy/public/sitedown.html index 28019d4c..d397ea11 100644 --- a/proxy/public/sitedown.html +++ b/proxy/public/sitedown.html @@ -3,7 +3,7 @@ - + 503 Site Temporarily Down diff --git a/proxy/public/template.html b/proxy/public/template.html index b2794b72..a0863c8a 100644 --- a/proxy/public/template.html +++ b/proxy/public/template.html @@ -3,7 +3,7 @@ - + Error 404 - Catalog diff --git a/tools/harvest_source_import/dev-requirements.txt b/tools/harvest_source_import/dev-requirements.txt new file mode 100644 index 00000000..b778a182 --- /dev/null +++ b/tools/harvest_source_import/dev-requirements.txt @@ -0,0 +1,5 @@ +-r requirements.txt +pytest>=5.4.2 +pytest-vcr>=1.0.2 +flake8>=3.8.1 +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/tools/harvest_source_import/requirements.txt b/tools/harvest_source_import/requirements.txt new file mode 100644 index 00000000..9fb26a94 --- /dev/null +++ b/tools/harvest_source_import/requirements.txt @@ -0,0 +1,4 @@ +requests>=2.32.0 +pytest>=5.4.2 +pytest-vcr>=1.0.2 +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/vars.staging.yml b/vars.staging.yml new file mode 100644 index 00000000..a42dc319 --- /dev/null +++ b/vars.staging.yml @@ -0,0 +1,65 @@ +# This is the name to use for the staging catalog app in Cloud Foundry +app_name: catalog +space_name: staging + +ckanext__saml2auth__entity_id: urn:gov:gsa:SAML:2.0.profiles:sp:sso:gsa:datagov-stage-catalog +ckanext__saml2auth__idp_metadata__local_path: ckan/setup/login.production.idp.xml + +web-instances: 2 +admin-instances: 1 +gather-instances: 1 +fetch-instances: 1 +proxy-instances: 1 +memory_quota: 750M +gather_memory_quota: 3G + +new_relic_monitor_mode: true + +# use CDN domain for route-public if available, otherwise use route-external +route-public: catalog-stage.data.gov +route-external: catalog-stage-datagov.app.cloud.gov +route-internal: catalog-stage-datagov.apps.internal +route-external-admin: catalog-stage-admin-datagov.app.cloud.gov +route-internal-admin: catalog-stage-admin-datagov.apps.internal + +saml2_certificate: | + -----BEGIN CERTIFICATE----- + MIIGQTCCBCkCFAHCH4OnnkSn89liqLUTPL4V4tJkMA0GCSqGSIb3DQEBCwUAMIHc + MQswCQYDVQQGEwJVUzEdMBsGA1UECAwURGlzdHJpY3Qgb2YgQ29sdW1iaWExEzAR + BgNVBAcMCldhc2hpbmd0b24xKDAmBgNVBAoMH0dlbmVyYWwgU2VydmljZXMgQWRt + aW5pc3RyYXRpb24xKjAoBgNVBAsMIVRlY2hub2xvZ3kgVHJhbnNmb3JtYXRpb24g + U2VydmljZTEiMCAGCSqGSIb3DQEJARYTZGF0YWdvdmhlbHBAZ3NhLmdvdjEfMB0G + A1UEAwwWY2F0YWxvZy1zdGFnZS5kYXRhLmdvdjAeFw0yNDAzMDYxNjU4MTJaFw0y + NTA0MDUxNjU4MTJaMIHcMQswCQYDVQQGEwJVUzEdMBsGA1UECAwURGlzdHJpY3Qg + b2YgQ29sdW1iaWExEzARBgNVBAcMCldhc2hpbmd0b24xKDAmBgNVBAoMH0dlbmVy + YWwgU2VydmljZXMgQWRtaW5pc3RyYXRpb24xKjAoBgNVBAsMIVRlY2hub2xvZ3kg + VHJhbnNmb3JtYXRpb24gU2VydmljZTEiMCAGCSqGSIb3DQEJARYTZGF0YWdvdmhl + bHBAZ3NhLmdvdjEfMB0GA1UEAwwWY2F0YWxvZy1zdGFnZS5kYXRhLmdvdjCCAiIw + DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKuF+/5v1Hkor4avTvSvqJVQ/naU + 6sVsOU2tSwfZcjo2ERDsiPgemude7nOhvZ5WgQakmPb13iS2nHOADG1UF0x4iYNF + LYnoxOt+LphTkEnFFkkbV0tUYzZdDegOjfpxbjZk7/aa8q74UOLbrnjkspNLdWlL + JDDZ4sPkIuVnGMIz033oVOZO9xZq1bGrwDEqmm8pH572qB/CJUnyG9gJBauzcgMZ + zcy+Np8bAQdp799kIE40MPbCS6gMYg6+Lg0PqDobe+LNjGv/yFEYx9oQ/dCpheoU + wtTyqOeSR+oq1GN8Ebt0AxccDVDdbLg/ciXbR9ID29DhsVtYeYpY2D60TnpVvd26 + bo7NgOa3qhsNZp+mAXmpG4G3aV9nJr5mmq9bYpSlXRjvtkSRY+N3TS0nX9xim60u + HDhnUFaNCOC8bIYsO8fdLUJ8hUc9SiyobCvb+sKUObTBnxpSDr4CcPwyAB65ElMe + O//ezP3vOnlc03RBk1ZCqezdxI0RmzqsoHxUFIBXFjeisCzs3OkkHln7uydTpBU9 + MJbsunPRLMu7AkAVX5o3k6SLAupyFY6pcJoXOgF0Rx0CcxN6tGHGJL+e8ttUk5Oo + i+2ZUUXJFoxkSi6xQQV7FzNnTjfRJWaJC2xMz7ilm1bbyueLzQ3MqxYRyA5mzCt3 + JaisICaDYvo8sqjBAgMBAAEwDQYJKoZIhvcNAQELBQADggIBAAyuURjrp+/89iPf + 1JUVuvpB5GiblpbIo7d1Wvcu8DgqXMcc8OclNWyqJNnI07VtumnOVUUpK/FowU3B + hiSSMX/PWUQtjlEIBMH84MfAM++tYiSfUEK/yY8zoAaL2J0FEca7QQ+f3Ju5h+qU + w/x1rJ9WTgms/okvnjNTQxTsHnDmXvzXbmeNJjs2etRSWQOdOqGuM4SCosr3nr1b + uXJWUL7T8OZtM1hQ3xB2GKG2tNbM1lJ0Q9++65k/twXf8pUgIIXqv7I6FReOznxS + EqwMl7VBCtBrkwdxbnceirfmtxUWleH6wWH0b4snwXAdXQ8hHhJRH1G53JliRKKr + svSExyW6z8eCuA73LqFyQmhRM2PP306Cwyb4pYcVazA/zQlgefUbnbXru5nZ5aRT + E2l38PTrNou9UMFGxZ6DxXPbEPnceYCYI7cPgs/9Cl5nM99FjHg22KQOpteKObEg + kodTrEC4Ge7yZ6lsobhKy884v+fCh85deEbfYKqZh63nLWynlbNcOk0BOskGKb7l + XixzZRTin/Rg3BkMUzKcE/M2z0W2o5GyKcUkgpj/rCUKHWkAdVGYZD8CM3NCCEY3 + MJafXHZnY6PIE2glDU7sDRfAtvK2aLEfXnsRXLSUefqpQdNSJ1WTkGn770gHpEWk + qwTG1ciJBV6hcOXG2UJv50oGC4t/ + -----END CERTIFICATE----- + +googleanalytics_id: UA-00000000-2 + +deny_package_create: false