From 37bb1bfb5d19d2d37ebcd5de998ee2b3a235d101 Mon Sep 17 00:00:00 2001 From: ajnisbet Date: Tue, 22 Aug 2023 11:05:35 -0700 Subject: [PATCH] Dependency upgrade --- Makefile | 6 +++--- docker/apple-silicon.Dockerfile | 5 +++-- example-config.yaml | 4 ---- opentopodata/backend.py | 16 +--------------- opentopodata/config.py | 2 -- requirements.in | 1 + requirements.txt | 7 +++++-- 7 files changed, 13 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index b8a715f..12db43f 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,10 @@ daemon: docker run --rm -itd --volume "$(shell pwd)/data:/app/data:ro" -p 5000:5000 opentopodata:$(VERSION) test: build black-check - docker run --rm -e DISABLE_MEMCACHE=1 --volume "$(shell pwd)/htmlcov:/app/htmlcov" opentopodata:$(VERSION) pytest --ignore=data --ignore=scripts --cov=opentopodata --cov-report html + docker run --rm -e DISABLE_MEMCACHE=1 --volume "$(shell pwd)/htmlcov:/app/htmlcov" opentopodata:$(VERSION) python -m pytest --ignore=data --ignore=scripts --cov=opentopodata --cov-report html --timeout=10 test-m1: build-m1 black-check - docker run --rm -e DISABLE_MEMCACHE=1 --volume "$(shell pwd)/htmlcov:/app/htmlcov" opentopodata:$(VERSION) pytest --ignore=data --ignore=scripts --cov=opentopodata --cov-report html + docker run --rm -e DISABLE_MEMCACHE=1 --volume "$(shell pwd)/htmlcov:/app/htmlcov" opentopodata:$(VERSION) python -m pytest --ignore=data --ignore=scripts --cov=opentopodata --cov-report html --timeout=10 run-local: FLASK_APP=opentopodata/api.py FLASK_DEBUG=1 flask run --port 5000 @@ -26,7 +26,7 @@ black: black --target-version py39 tests opentopodata black-check: - docker run --rm opentopodata:$(VERSION) black --check --target-version py39 tests opentopodata + docker run --rm opentopodata:$(VERSION) python -m black --check --target-version py39 tests opentopodata update-requirements: build # pip-compile gets confused if there's already a requirements.txt file, and diff --git a/docker/apple-silicon.Dockerfile b/docker/apple-silicon.Dockerfile index 9c21ac3..9f81059 100644 --- a/docker/apple-silicon.Dockerfile +++ b/docker/apple-silicon.Dockerfile @@ -5,7 +5,8 @@ # It works just the same as the main image, but is much larger and slower to # build. -FROM osgeo/gdal:ubuntu-full-3.5.2 +FROM ghcr.io/osgeo/gdal:ubuntu-full-3.6.4 +RUN python --version RUN set -e && \ apt-get update && \ apt-get install -y --no-install-recommends \ @@ -16,7 +17,7 @@ RUN set -e && \ g++ \ supervisor \ libmemcached-dev \ - python3.8-dev && \ + python3.10-dev && \ rm -rf /var/lib/apt/lists/* COPY requirements.txt /app/requirements.txt diff --git a/example-config.yaml b/example-config.yaml index 4262c25..6e5014a 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -11,10 +11,6 @@ max_locations_per_request: 100 access_control_allow_origin: "*" -# Use mmap to cache files for faster repeated reads on slow networked filesystems. -# See https://github.com/ajnisbet/opentopodata/pull/74 -read_with_mmap: false - datasets: # A small testing dataset is included in the repo. diff --git a/opentopodata/backend.py b/opentopodata/backend.py index 8a5ace5..d5a6c8f 100644 --- a/opentopodata/backend.py +++ b/opentopodata/backend.py @@ -1,5 +1,4 @@ import collections -import mmap from rasterio.enums import Resampling import numpy as np @@ -86,16 +85,8 @@ def _get_elevation_from_path(lats, lons, path, interpolation, use_mmap=False): lons = np.asarray(lons) lats = np.asarray(lats) - f_mmap = None - dataset = None - try: - if use_mmap: - f_mmap = open(path, "rb") - dataset = mmap.mmap(f_mmap.fileno(), length=0, access=mmap.ACCESS_READ) - else: - dataset = path - with rasterio.open(dataset) as f: + with rasterio.open(path) as f: if f.crs is None: msg = "Dataset has no coordinate reference system." msg += f" Check the file '{path}' is a geo raster." @@ -162,11 +153,6 @@ def _get_elevation_from_path(lats, lons, path, interpolation, use_mmap=False): msg += " and that the file is not corrupt." raise InputError(msg) raise e - finally: - if isinstance(dataset, mmap.mmap): - dataset.close() - if f_mmap: - f_mmap.close() return z_all diff --git a/opentopodata/config.py b/opentopodata/config.py index ac94fad..9ccf99d 100644 --- a/opentopodata/config.py +++ b/opentopodata/config.py @@ -21,7 +21,6 @@ "dataset.filename_tile_size": 1, "dataset.filename_epsg": utils.WGS84_LATLON_EPSG, "access_control_allow_origin": None, - "read_with_mmap": False, } @@ -148,7 +147,6 @@ def load_config(): config["access_control_allow_origin"] = config.get( "access_control_allow_origin", DEFAULTS["access_control_allow_origin"] ) - config["read_with_mmap"] = config.get("read_with_mmap", DEFAULTS["read_with_mmap"]) # Validate CORS. Must have protocol, domain, and optionally port. _validate_cors(config["access_control_allow_origin"]) diff --git a/requirements.in b/requirements.in index 8cf5e93..4e3052b 100644 --- a/requirements.in +++ b/requirements.in @@ -9,6 +9,7 @@ pylibmc pyproj pytest pytest-cov +pytest-timeout PyYAML rasterio<1.3.0 requests diff --git a/requirements.txt b/requirements.txt index 153eedd..19192ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,7 @@ coverage[toml]==7.3.0 # via pytest-cov exceptiongroup==1.1.3 # via pytest -flask==2.3.2 +flask==2.3.3 # via # -r requirements.in # flask-caching @@ -95,8 +95,11 @@ pytest==7.4.0 # via # -r requirements.in # pytest-cov + # pytest-timeout pytest-cov==4.1.0 # via -r requirements.in +pytest-timeout==2.1.0 + # via -r requirements.in pyyaml==6.0.1 # via -r requirements.in rasterio==1.2.10 @@ -119,7 +122,7 @@ urllib3==2.0.4 # via requests werkzeug==2.3.7 # via flask -wheel==0.41.1 +wheel==0.41.2 # via pip-tools zipp==3.16.2 # via importlib-metadata