diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c236840 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.venv +skaffold.env +.git +.github +./polytope_server.egg-info +**/.git +./polytope-mars/tests +./polytope/tests +./polytope/examples +./polytope/performance \ No newline at end of file diff --git a/.gitignore b/.gitignore index eaaeec7..12e9167 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,9 @@ htmlcov validated.yaml merged.yaml polytope_server.egg-info -**/build \ No newline at end of file +**/build +.venv +skaffold.env +polytope-mars +polytope +covjsonkit \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6a0cd81 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,390 @@ +## +## Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +## In applying this licence, ECMWF does not waive the privileges and immunities +## granted to it by virtue of its status as an intergovernmental organisation nor +## does it submit to any jurisdiction. +## + +ARG fdb_base=blank-base +ARG mars_base_c=blank-base +ARG mars_base_cpp=blank-base +ARG gribjump_base=blank-base + +####################################################### +# C O M M O N +# based on alpine linux +####################################################### + +FROM python:3.11-alpine AS polytope-common + +ARG HOME_DIR=/home/polytope +ARG developer_mode + +# Install build dependencies +RUN apk add --no-cache --virtual .build-deps gcc musl-dev openldap-dev curl \ + && apk add --no-cache openldap + +# Create user and group +RUN set -eux \ + && addgroup --system polytope --gid 474 \ + && adduser --system polytope --ingroup polytope --home ${HOME_DIR} \ + && mkdir -p ${HOME_DIR}/polytope-server \ + && chown -R polytope:polytope ${HOME_DIR} + +# Switch to user polytope +USER polytope + +WORKDIR ${HOME_DIR}/polytope-server + +# Copy requirements.txt with correct ownership +COPY --chown=polytope:polytope ./requirements.txt $PWD + +# Install uv in user space +RUN pip install --user uv + +# **Update PATH to include virtual environment and user local bin** +# This makes sure that the default python and pip commands +# point to the versions in the virtual environment. +ENV PATH="${HOME_DIR}/.venv/bin:${HOME_DIR}/.local/bin:${PATH}" + +# Create a virtual environment +RUN uv venv ${HOME_DIR}/.venv + +# Install requirements +RUN uv pip install -r requirements.txt + +# Copy the rest of the application code +COPY --chown=polytope:polytope . $PWD + +RUN set -eux \ + && if [ $developer_mode = true ]; then \ + uv pip install ./polytope-mars ./polytope ./covjsonkit; \ + fi + +# Install the application +RUN uv pip install --upgrade . + +# Clean up build dependencies to reduce image size +USER root +RUN apk del .build-deps + + +# Switch back to polytope user +USER polytope + + +####################################################### +# N O O P I M A G E +####################################################### +FROM python:3.11-bookworm AS blank-base +# create blank directories to copy from in the final stage, optional dependencies aren't built +RUN set -eux \ + && mkdir -p /root/.local \ + && mkdir -p /opt/ecmwf/mars-client \ + && mkdir -p /opt/ecmwf/mars-client-cpp \ + && mkdir -p /opt/ecmwf/mars-client-cloud \ + && mkdir -p /opt/fdb \ + && mkdir -p /opt/fdb-gribjump \ + && touch /usr/local/bin/mars + +####################################################### +# F D B B U I L D +####################################################### +FROM python:3.11-bookworm AS fdb-base +RUN apt update +# COPY polytope-deployment/common/default_fdb_schema /polytope/config/fdb/default + +# Install FDB from open source repositories +RUN set -eux && \ + apt install -y cmake gnupg build-essential libtinfo5 net-tools libnetcdf19 libnetcdf-dev bison flex && \ + rm -rf source && \ + rm -rf build && \ + mkdir -p source && \ + mkdir -p build && \ + mkdir -p /opt/fdb/ + +# Download ecbuild +RUN set -eux && \ + git clone --depth 1 --branch 3.8.2 https://github.com/ecmwf/ecbuild.git /ecbuild + +ENV PATH=/ecbuild/bin:$PATH + +# Install eckit +RUN set -eux && \ + git clone --depth 1 --branch develop https://github.com/ecmwf/eckit.git /source/eckit && \ + cd /source/eckit && git checkout develop && \ + mkdir -p /build/eckit && \ + cd /build/eckit && \ + ecbuild --prefix=/opt/fdb -- -DCMAKE_PREFIX_PATH=/opt/fdb /source/eckit && \ + make -j4 && \ + make install + +# Install eccodes +RUN set -eux && \ + git clone --depth 1 --branch 2.33.1 https://github.com/ecmwf/eccodes.git /source/eccodes && \ + mkdir -p /build/eccodes && \ + cd /build/eccodes && \ + ecbuild --prefix=/opt/fdb -DENABLE_FORTRAN=OFF -- -DCMAKE_PREFIX_PATH=/opt/fdb /source/eccodes && \ + make -j4 && \ + make install + +# Install metkit +RUN set -eux && \ + git clone --depth 1 --branch develop https://github.com/ecmwf/metkit.git /source/metkit && \ + cd /source/metkit && git checkout develop && \ + mkdir -p /build/metkit && \ + cd /build/metkit && \ + ecbuild --prefix=/opt/fdb -- -DCMAKE_PREFIX_PATH=/opt/fdb /source/metkit && \ + make -j4 && \ + make install + +# Install fdb \ +RUN set -eux && \ + git clone --depth 1 --branch develop https://github.com/ecmwf/fdb.git /source/fdb && \ + cd /source/fdb && git checkout develop && \ + mkdir -p /build/fdb && \ + cd /build/fdb && \ + ecbuild --prefix=/opt/fdb -- -DCMAKE_PREFIX_PATH="/opt/fdb;/opt/fdb/eckit;/opt/fdb/metkit" /source/fdb && \ + make -j4 && \ + make install + +RUN set -eux && \ + rm -rf /source && \ + rm -rf /build + +ARG ssh_prv_key +ARG ssh_pub_key + +# Install pyfdb \ +RUN set -eux \ + && git clone --single-branch --branch 0.0.3 https://github.com/ecmwf/pyfdb.git \ + && python -m pip install ./pyfdb --user + +####################################################### +# G R I B J U M P B U I L D +####################################################### + +FROM python:3.11-bookworm AS gribjump-base +ARG rpm_repo + +RUN response=$(curl -s -w "%{http_code}" ${rpm_repo}) \ + && if [ "$response" = "403" ]; then echo "Unauthorized access to ${rpm_repo} "; fi + +RUN set -eux \ + && apt-get update \ + && apt-get install -y gnupg2 curl ca-certificates \ + && curl -fsSL "${rpm_repo}/private-raw-repos-config/debian/bookworm/stable/public.gpg.key" | gpg --dearmor -o /usr/share/keyrings/mars-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/mars-archive-keyring.gpg] ${rpm_repo}/private-debian-bookworm-stable/ bookworm main" | tee /etc/apt/sources.list.d/mars.list + +RUN set -eux \ + && apt-get update \ + && apt install -y gribjump-client=0.5.4-gribjump + +RUN set -eux \ + ls -R /opt + +# gribjump not open source yet, clone it. +RUN set -eux \ + && git clone --single-branch --branch develop https://github.com/ecmwf/gribjump.git +# Install pygribjump +RUN set -eux \ +&& cd /gribjump \ + && python -m pip install . --user \ + && rm -rf /gribjump + +####################################################### +# F D B R E M O T E B U I L D +####################################################### + +FROM fdb-base AS fdb-remote-base + +ARG ssh_prv_key +ARG ssh_pub_key + +# gribjump not open source yet, clone it. +# COPY fdbremote /source/fdbremote +# Install fdb from local source (need a private version from internal ECMWF repository) +RUN apt-get install -y gfortran +RUN set -eux \ + # Configure SSH for private repository access + && mkdir -p /root/.ssh \ + && echo "$ssh_prv_key" > /root/.ssh/id_rsa \ + && echo "$ssh_pub_key" > /root/.ssh/id_rsa.pub \ + && chmod 600 /root/.ssh/id_rsa \ + && chmod 600 /root/.ssh/id_rsa.pub \ + && echo "StrictHostKeyChecking=no" > /root/.ssh/config \ + && mkdir -p build \ + && mkdir -p source \ + && mkdir -p /build/fdb \ + && cd /build/fdb \ + && ecbuild --prefix=/opt/fdbremote -- /source/fdbremote \ + && make -j4 \ + && make install \ + && rm -rf /build \ + && rm -rf /source \ + && rm -rf /root/.ssh + +####################################################### +# M A R S B A S E +####################################################### +FROM python:3.11-bookworm AS mars-base +ARG rpm_repo +ARG mars_client_cpp_version + +RUN response=$(curl -s -w "%{http_code}" ${rpm_repo}) \ + && if [ "$response" = "403" ]; then echo "Unauthorized access to ${rpm_repo} "; fi + +RUN set -eux \ + && curl -o stable-public.gpg.key "${rpm_repo}/private-raw-repos-config/debian/bookworm/stable/public.gpg.key" \ + && echo "deb ${rpm_repo}/private-debian-bookworm-stable/ bookworm main" >> /etc/apt/sources.list \ + && apt-key add stable-public.gpg.key \ + && apt-get update \ + && apt install -y libnetcdf19 liblapack3 + +FROM mars-base AS mars-base-c +RUN apt update && apt install -y liblapack3 mars-client=6.33.20.2 mars-client-cloud + +FROM mars-base AS mars-base-cpp +RUN apt update && apt install -y mars-client-cpp=${mars_client_cpp_version} +RUN set -eux \ + && python3 -m pip install git+https://github.com/ecmwf/pyfdb.git@master --user + +FROM blank-base AS blank-base-c +FROM blank-base AS blank-base-cpp + +####################################################### +# S W I T C H B A S E I M A G E S +####################################################### + +FROM ${fdb_base} AS fdb-base-final + +FROM ${mars_base_c} AS mars-c-base-final + +FROM ${mars_base_cpp} AS mars-cpp-base-final + +FROM ${gribjump_base} AS gribjump-base-final + + +####################################################### +# P Y T H O N R E Q U I R E M E N T S +####################################################### +FROM python:3.11-slim-bookworm AS worker-base +ARG developer_mode + +# contains compilers for building wheels which we don't want in the final image +RUN apt update +RUN apt-get install -y --no-install-recommends gcc libc6-dev make gnupg2 + +COPY ./requirements.txt /requirements.txt +RUN pip install uv --user +ENV PATH="/root/.venv/bin:/root/.local/bin:${PATH}" +RUN uv venv /root/.venv +RUN uv pip install -r requirements.txt +RUN uv pip install geopandas==1.0.1 + +COPY . ./polytope +RUN set -eux \ + && if [ $developer_mode = true ]; then \ + uv pip install ./polytope/polytope-mars ./polytope/polytope ./polytope/covjsonkit; \ + fi + +####################################################### +# W O R K E R +# based on debian bookworm +####################################################### + +FROM python:3.11-slim-bookworm AS worker + +ARG mars_config_branch +ARG mars_config_repo +ARG ssh_prv_key +ARG ssh_pub_key +ARG rpm_repo + + +RUN set -eux \ + && addgroup --system polytope --gid 474 \ + && adduser --system polytope --ingroup polytope --home /home/polytope \ + && mkdir /polytope && chmod -R o+rw /polytope + +RUN apt update \ + && apt install -y curl nano sudo ssh libgomp1 vim + +# Add polytope user to passwordless sudo group during build +RUN usermod -aG sudo polytope +RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +WORKDIR /polytope +USER polytope + + +# Copy MARS-related artifacts +COPY --chown=polytope ./aux/mars-wrapper.py /polytope/bin/mars-wrapper.py +COPY --chown=polytope ./aux/mars-wrapper-docker.py /polytope/bin/mars-wrapper-docker.py + +COPY --chown=polytope --from=mars-cpp-base-final /opt/ecmwf/mars-client-cpp /opt/ecmwf/mars-client-cpp +COPY --chown=polytope --from=mars-cpp-base-final /root/.local /home/polytope/.local +COPY --chown=polytope --from=mars-c-base-final /opt/ecmwf/mars-client /opt/ecmwf/mars-client +COPY --chown=polytope --from=mars-c-base-final /usr/local/bin/mars /usr/local/bin/mars +RUN sudo apt update \ + && sudo apt install -y libgomp1 git libnetcdf19 liblapack3 libfftw3-bin libproj25 \ + && sudo rm -rf /var/lib/apt/lists/* + + +# all of this is needed by the C client, would be nice to remove it at some point +RUN set -eux \ + && mkdir -p /home/polytope/.ssh \ + && chmod 0700 /home/polytope/.ssh \ + && ssh-keyscan git.ecmwf.int > /home/polytope/.ssh/known_hosts \ + && echo "$ssh_prv_key" > /home/polytope/.ssh/id_rsa \ + && echo "$ssh_pub_key" > /home/polytope/.ssh/id_rsa.pub \ + && chmod 600 /home/polytope/.ssh/id_rsa \ + && chmod 600 /home/polytope/.ssh/id_rsa.pub \ + && chmod 755 /polytope/bin/mars-wrapper.py \ + && chmod 755 /polytope/bin/mars-wrapper-docker.py + +ENV MARS_CONFIGS_REPO=${mars_config_repo} +ENV MARS_CONFIGS_BRANCH=${mars_config_branch} +ENV PATH="/polytope/bin/:/opt/ecmwf/mars-client/bin:/opt/ecmwf/mars-client-cloud/bin:${PATH}" + +# Copy FDB-related artifacts +COPY --chown=polytope --from=fdb-base-final /opt/fdb/ /opt/fdb/ +COPY --chown=polytope ./aux/default_fdb_schema /polytope/config/fdb/default +RUN mkdir -p /polytope/fdb/ && sudo chmod -R o+rw /polytope/fdb +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/fdb/lib:/opt/ecmwf/gribjump-client/lib +COPY --chown=polytope --from=fdb-base-final /root/.local /home/polytope/.local + +# COPY --chown=polytope --from=fdb-remote-base /opt/fdbremote/ /opt/fdbremote/ + +# Copy gribjump-related artifacts, including python libraries +# COPY --chown=polytope --from=gribjump-base-final /opt/fdb/ /opt/fdb/ +COPY --chown=polytope --from=gribjump-base-final /opt/ecmwf/gribjump-client/ /opt/ecmwf/gribjump-client/ +COPY --chown=polytope --from=gribjump-base-final /root/.local /home/polytope/.local +# RUN sudo apt install -y libopenjp2-7 +# COPY polytope-deployment/common/default_fdb_schema /polytope/config/fdb/default + +# Copy python requirements +COPY --chown=polytope --from=worker-base /root/.venv /home/polytope/.local + + +# Install the server source +COPY --chown=polytope . /polytope/ + +RUN set -eux \ + && mkdir /home/polytope/data + +# Remove itself from sudo group +RUN sudo deluser polytope sudo \ No newline at end of file diff --git a/admin/polytope-admin/polytope_admin/api/Auth.py b/admin/polytope-admin/polytope_admin/api/Auth.py index 7a6cd40..b329189 100644 --- a/admin/polytope-admin/polytope_admin/api/Auth.py +++ b/admin/polytope-admin/polytope_admin/api/Auth.py @@ -150,12 +150,12 @@ def fetch_key(self, login=True): email = self.read_email self._logger.info("Polytope user key found in session cache for user " + config["username"]) else: - key_file = Path(config["key_path"]) / config["username"] + key_file = Path(config["key_path"]) try: with open(str(key_file), "r") as infile: info = json.load(infile) - key = info["key"] - email = info["email"] + key = info["user_key"] + email = info["user_email"] except FileNotFoundError: key = None email = None @@ -190,7 +190,7 @@ def persist(self, key, email, username=None): if not username: username = config["username"] os.makedirs(config["key_path"], exist_ok=True) - key_file = Path(config["key_path"]) / username + key_file = Path(config["key_path"]) with open(str(key_file), "w", encoding="utf8") as outfile: json.dump({"key": key, "email": email}, outfile) self.read_key = key @@ -214,7 +214,7 @@ def erase(self, username=None): config = self.config.get() if not username: username = config["username"] - key_path = Path(config["key_path"]) / username + key_path = Path(config["key_path"]) try: os.remove(str(key_path)) self._logger.info("Credentials removed for " + username) diff --git a/aux/default_fdb_schema b/aux/default_fdb_schema new file mode 100644 index 0000000..625f87e --- /dev/null +++ b/aux/default_fdb_schema @@ -0,0 +1,590 @@ +# From https://github.com/ecmwf/fdb/blob/c529bf5293d1f5e33048b1b12e7fdfbf4d7448b2/tests/fdb/etc/fdb/schema + + +# * Format of the rules is: + +# [a1, a2, a3 ...[b1, b2, b3... [c1, c2, c3...]]] + +# - The first level (a) defines which attributes are used to name the top level directory +# - The second level (b) defines which attributes are used to name the data files +# - The third level (c) defines which attributes are used as index keys + +# * Rules can be grouped + +# [a1, a2, a3 ... +# [b1, b2, b3... [c1, c2, c3...]] +# [B1, B2, B3... [C1, C2, C3...]] +# ] + +# * A list of values can be given for an attribute +# [ ..., stream=enfo/efov, ... ] +# This will be used when matching rules. + +# * Attributes can be typed +# Globally, at the begining of this file: + +# refdate: Date; + +# or in the context of a rule: +# [type=cl, ... [date:ClimateMonth, ...]] + +# Typing attributes is done when the user's requests or the GRIB values need to be modified before directories, files and indexes are created. For example, ClimateMonth will transform 2010-04-01 to 'may' internally. + +# * Attributes can be optional +# [ step, levelist?, param ] +# They will be replaced internally by an empty value. It is also posiible to provide a default subtitution value: e.g. [domain?g] will consider the domain to be 'g' if missing. + +# * Attributes can be removed: +# [grid-] +# This is useful to remove attributes present in the GRIB that should not be ignored + +# * Rules are matched: + +# - If the attributes are present in the GRIB/Request, or marked optional or ignored +# - If a list of possible value is provided, one of them must match, for example +# [ class, expver, stream=enfo/efov, date, time, domain ] +# will match either stream=enfo or stream=efov, all other attributes will be matched if they exist in the GRIB or user's request + +# * On archive: +# - Attributes are extracted from the GRIB (namespace 'mars'), possibly modified by the attribute type +# - Only the first rule is used, so order is important +# - All GRIB attributes must be used by the rules, otherwise an error is raised + +# * On retrieve: +# - Attributes are extracted from the user's request, possibly modified by the attribute type (e.g. for handling of U/V) +# - All the matching rules are considered +# - Only attributes listed in the rules are used to extract values from the user's request + + +# Default types + +param: Param; +step: Step; +date: Date; +hdate: Date; +refdate: Date; +latitude: Double; +longitude: Double; +levelist: Double; +grid: Grid; +expver: Expver; + +time: Time; +fcmonth: Integer; + +number: Integer; +frequency: Integer; +direction: Integer; +channel: Integer; + +instrument: Integer; +ident: Integer; + +diagnostic: Integer; +iteration: Integer; +system: Integer; +method: Integer; + +# ??????? + +# reference: Integer; +# fcperiod: Integer; + +# opttime: Integer; +# leadtime: Integer; + +# quantile: ?????? +# range: ?????? + +# band: Integer; + + +######################################################## +# These rules must be first, otherwise fields of These +# classes will be index with the default rule for oper +[ class=ti/s2, expver, stream, date, time, model + [ origin, type, levtype, hdate? + [ step, number?, levelist?, param ]] +] + +[ class=ms, expver, stream, date, time, country=de + [ domain, type, levtype, dbase, rki, rty, ty + [ step, levelist?, param ]] +] + +[ class=ms, expver, stream, date, time, country=it + [ domain, type, levtype, model, bcmodel, icmodel:First3 + [ step, levelist?, param ] + ] +] + +[ class=el, expver, stream, date, time, domain + [ origin, type, levtype + [ step, levelist?, param ]] +] + +######################################################## +# The are the rules matching most of the fields +# oper/dcda +[ class, expver, stream=oper/dcda/scda, date, time, domain? + + [ type=im/sim + [ step?, ident, instrument, channel ]] + + [ type=ssd + [ step, param, ident, instrument, channel ]] + + [ type=4i, levtype + [ step, iteration, levelist, param ]] + + [ type=me, levtype + [ step, number, levelist?, param ]] + + [ type=ef, levtype + [ step, levelist?, param, channel? ]] + + [ type=ofb/mfb + [ obsgroup, reportype ]] + + [ type, levtype + [ step, levelist?, param ]] + +] + +# dcwv/scwv/wave +[ class, expver, stream=dcwv/scwv/wave, date, time, domain + [ type, levtype + [ step, param, frequency?, direction? ]]] + +# enfo +[ class, expver, stream=enfo/efov, date, time, domain + + [ type, levtype=dp, product?, section? + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type=tu, levtype, reference + [ step, number, levelist?, param ]] + + [ type, levtype + [ step, quantile?, number?, levelist?, param ]] + +] + +# waef/weov +[ class, expver, stream=waef/weov, date, time, domain + [ type, levtype + [ step, number?, param, frequency?, direction? ]] +] + +######################################################## +# enda +[ class, expver, stream=enda, date, time, domain + + [ type=ef/em/es/ses, levtype + [ step, number?, levelist?, param, channel? ]] + + [ type=ssd + [ step, number, param, ident, instrument, channel ]] + + + [ type, levtype + [ step, number?, levelist?, param ]] +] + +# ewda +[ class, expver, stream=ewda, date, time, domain + [ type, levtype + [ step, number?, param, frequency?, direction? ]] +] + + +######################################################## +# elda +[ class, expver, stream=elda, date, time, domain? + + [ type=ofb/mfb + [ obsgroup, reportype ]] + + [ type, levtype, anoffset + [ step, number?, levelist?, iteration?, param, channel? ]] +] + +# ewda +[ class, expver, stream=ewla, date, time, domain + [ type, levtype, anoffset + [ step, number?, param, frequency?, direction? ]] +] + +######################################################## +# elda +[ class, expver, stream=lwda, date, time, domain? + + [ type=ssd, anoffset + [ step, param, ident, instrument, channel ]] + + [type=me, levtype, anoffset + [ number, step, levelist?, param]] + + [ type=4i, levtype, anoffset + [ step, iteration, levelist, param ]] + + [ type=ofb/mfb + [ obsgroup, reportype ]] + + [ type, levtype, anoffset + [ step, levelist?, param]] +] + +# ewda +[ class, expver, stream=lwwv, date, time, domain + [ type, levtype, anoffset + [ step, param, frequency?, direction? ]] +] +######################################################## +# amap +[ class, expver, stream=amap, date, time, domain + [ type, levtype, origin + [ step, levelist?, param ]]] + +# maed +[ class, expver, stream=maed, date, time, domain + [ type, levtype, origin + [ step, levelist?, param ]]] + +# mawv +[ class, expver, stream=mawv, date, time, domain + [ type, levtype, origin + [ step, param, frequency?, direction? ]]] + +# cher +[ class, expver, stream=cher, date, time, domain + [ type, levtype + [ step, levelist, param ]]] + + +# efhc +[ class, expver, stream=efhc, refdate, time, domain + [ type, levtype, date + [ step, number?, levelist?, param ]]] + +# efho +[ class, expver, stream=efho, date, time, domain + [ type, levtype, hdate + [ step, number?, levelist?, param ]]] + + +# efhs +[ class, expver, stream=efhs, date, time, domain + [ type, levtype + [ step, quantile?, number?, levelist?, param ]]] + +# wehs +[ class, expver, stream=wehs, date, time, domain + [ type, levtype + [ step, quantile?, number?, levelist?, param ]]] + +# kwbc +[ class, expver, stream=kwbc, date, time, domain + [ type, levtype + [ step, number?, levelist?, param ]]] + +# ehmm +[ class, expver, stream=ehmm, date, time, domain + [ type, levtype, hdate + [ fcmonth, levelist?, param ]]] + + +# ammc/cwao/edzw/egrr/lfpw/rjtd/toga +[ class, expver, stream=ammc/cwao/edzw/egrr/lfpw/rjtd/toga/fgge, date, time, domain + [ type, levtype + [ step, levelist?, param ]]] + +######################################################################## + +# enfh +[ class, expver, stream=enfh, date, time, domain + + [ type, levtype=dp, hdate, product?, section? + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, hdate + [ step, number?, levelist?, param ]] +] + +# enwh +[ class, expver, stream=enwh, date, time, domain + [ type, levtype, hdate + [ step, number?, param, frequency?, direction? ]] +] + +######################################################################## +# sens +[ class, expver, stream=sens, date, time, domain + [ type, levtype + [ step, diagnostic, iteration, levelist?, param ]]] + +######################################################################## +# esmm +[ class, expver, stream=esmm, date, time, domain + [ type, levtype + [ fcmonth, levelist?, param ]]] +# ewhc +[ class, expver, stream=ewhc, refdate, time, domain + [ type, levtype, date + [ step, number?, param, frequency?, direction? ]]] + +######################################################################## +# ewho +[ class, expver, stream=ewho, date, time, domain + [ type, levtype, hdate + [ step, number?, param, frequency?, direction? ]]] + +# mfam +[ class, expver, stream=mfam, date, time, domain + + [ type=pb/pd, levtype, origin, system?, method + [ fcperiod, quantile, levelist?, param ]] + + [ type, levtype, origin, system?, method + [ fcperiod, number?, levelist?, param ]] + +] + +# mfhm +[ class, expver, stream=mfhm, refdate, time, domain + [ type, levtype, origin, system?, method, date? + [ fcperiod, number?, levelist?, param ]]] +# mfhw +[ class, expver, stream=mfhw, refdate, time, domain + [ type, levtype, origin, system?, method, date + [ step, number?, param ]]] +# mfwm +[ class, expver, stream=mfwm, date, time, domain + [ type, levtype, origin, system?, method + [ fcperiod, number, param ]]] +# mhwm +[ class, expver, stream=mhwm, refdate, time, domain + [ type, levtype, origin, system?, method, date + [ fcperiod, number, param ]]] + +# mmsf +[ class, expver, stream=mmsf, date, time, domain + + [ type, levtype=dp, origin, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, origin, system?, method + [ step, number, levelist?, param ]] +] + +# mnfc +[ class, expver, stream=mnfc, date, time, domain + + [ type, levtype=dp, origin, product, section, system?, method + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, origin, system?, method + [ step, number?, levelist?, param ]] +] + +# mnfh +[ class, expver, stream=mnfh, refdate, time, domain + [ type, levtype=dp, origin, product, section, system?, method, date + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + [ type, levtype, origin, system?, method, date? + [ step, number?, levelist?, param ]] +] + +# mnfm +[ class, expver, stream=mnfm, date, time, domain + [ type, levtype, origin, system?, method + [ fcperiod, number?, levelist?, param ]]] + +# mnfw +[ class, expver, stream=mnfw, date, time, domain + [ type, levtype, origin, system?, method + [ step, number?, param ]]] + +# ea/mnth +[ class=ea, expver, stream=mnth, date, domain + [ type, levtype + [ time, step?, levelist?, param ]]] + +# mnth +[ class, expver, stream=mnth, domain + [ type=cl, levtype + [ date: ClimateMonthly, time, levelist?, param ]] + [ type, levtype + [ date , time, step?, levelist?, param ]]] + +# mofc +[ class, expver, stream=mofc, date, time, domain + [ type, levtype=dp, product, section, system?, method + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + [ type, levtype, system?, method + [ step, number?, levelist?, param ]] +] + +# mofm +[ class, expver, stream=mofm, date, time, domain + [ type, levtype, system?, method + [ fcperiod, number, levelist?, param ]]] + +# mmsa/msmm +[ class, expver, stream=mmsa, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number?, levelist?, param ]]] + +[ class, expver, stream=msmm, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number?, levelist?, param ]]] + +# ocea +[ class, expver, stream=ocea, date, time, domain + [ type, levtype, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] +] + +#=# seas +[ class, expver, stream=seas, date, time, domain + + [ type, levtype=dp, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, system?, method + [ step, number, levelist?, param ]] +] + +# sfmm/smma +[ class, expver, stream=sfmm/smma, date, time, domain + [ type, levtype, system?, method + [ fcmonth, number?, levelist?, param ]]] + +# supd +[ class=od, expver, stream=supd, date, time, domain + [ type, levtype, origin?, grid + [ step, levelist?, param ]]] + +# For era +[ class, expver, stream=supd, date, time, domain + [ type, levtype, grid- # The minus sign is here to consume 'grid', but don't index it + [ step, levelist?, param ]]] + +# swmm +[ class, expver, stream=swmm, date, time, domain + [ type, levtype, system?, method + [ fcmonth, number, param ]]] + +# wamf +[ class, expver, stream=wamf, date, time, domain + [ type, levtype, system?, method + [ step, number?, param ]]] + +# ea/wamo +[ class=ea, expver, stream=wamo, date, domain + [ type, levtype + [ time, step?, param ]]] + +# wamo +[ class, expver, stream=wamo, domain + [ type=cl, levtype + [ date: ClimateMonthly, time, param ]] + [ type, levtype + [ date, time, step?, param ]]] + +# wamd +[ class, expver, stream=wamd, date, domain + [ type, levtype + [ param ]]] + +# wasf +[ class, expver, stream=wasf, date, time, domain + [ type, levtype, system?, method + [ step, number, param ]]] +# wmfm +[ class, expver, stream=wmfm, date, time, domain + [ type, levtype, system?, method + [ fcperiod, number, param ]]] + +# moda +[ class, expver, stream=moda, date, domain + [ type, levtype + [ levelist?, param ]]] + +# msdc/mdfa/msda +[ class, expver, stream=msdc/mdfa/msda, domain + [ type, levtype + [ date, time?, step?, levelist?, param ]]] + + + +# seap +[ class, expver, stream=seap, date, time, domain + [ type=sv/svar, levtype, origin, method? + [ step, leadtime, opttime, number, levelist?, param ]] + + [ type=ef, levtype, origin + [ step, levelist?, param, channel? ]] + + [ type, levtype, origin + [ step, levelist?, param ]] + + ] + +[ class, expver, stream=mmaf, date, time, domain + [ type, levtype, origin, system?, method + [ step, number, levelist?, param ]] +] + +[ class, expver, stream=mmam, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number, levelist?, param ]] +] + + +[ class, expver, stream=dacl, domain + [ type=pb, levtype + [ date: ClimateDaily, time, step, quantile, levelist?, param ]] + [ type, levtype + [ date: ClimateDaily, time, step, levelist?, param ]] + +] + +[ class, expver, stream=dacw, domain + [ type=pb, levtype + [ date: ClimateDaily, time, step, quantile, param ]] + [ type, levtype + [ date: ClimateDaily, time, step, param ]] + +] + +[ class, expver, stream=edmm/ewmm, date, time, domain + [ type=ssd + [ step, number, param, ident, instrument, channel ]] + [ type, levtype + [ step, number, levelist?, param ]] +] + +[ class, expver, stream=edmo/ewmo, date, domain + [ type, levtype + [ number, levelist?, param ]] +] + +# stream gfas +[ class=mc/rd, expver, stream=gfas, date, time, domain + [ type=ga, levtype + [ step, param ]] + + [ type=gsd + [ param, ident, instrument ]] + +] + +# class is e2 +[ class, expver, stream=espd, date, time, domain + [ type, levtype, origin, grid + [ step, number, levelist?, param ]]] + +[ class=cs, expver, stream, date:Default, time, domain + [ type, levtype + [ step, levelist?, param ]]] diff --git a/aux/mars-wrapper-docker.py b/aux/mars-wrapper-docker.py new file mode 100644 index 0000000..ec202d1 --- /dev/null +++ b/aux/mars-wrapper-docker.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. +# + + +import logging +import os +import subprocess +import sys + +import docker + + +def main(): + logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(name)s %(levelname)s %(message)s") + + assert len(sys.argv) == 2 + + c = docker.from_env() + # c = docker.DockerClient(base_url = ('tcp://' + + # os.environ['POLYTOPE_DOCKER_URL'])) + + container_name = os.environ["HOSTNAME"] + containers = c.containers.list() + cids = list(map(lambda x: x.short_id, containers)) + container = None + for cpos in range(len(cids)): + if container_name.startswith(cids[cpos]): + container = containers.pop(cpos) + break + if not container: + raise Exception("Container not found") + + container_port = str(os.environ.get("POLYTOPE_WORKER_MARS_LOCALPORT")) + external_port = container.ports[container_port + "/tcp"][0]["HostPort"] + swarm_node = container.labels["com.docker.swarm.node.id"] + nodes = c.nodes.list() + nids = list(map(lambda x: x.attrs["ID"], nodes)) + node = None + for npos in range(len(nids)): + if swarm_node == nids[npos]: + node = nodes.pop(npos) + break + if not node: + raise Exception("Node not found") + + node_name = node.attrs["Description"]["Hostname"] + + mars_command = os.environ.get("ECMWF_MARS_COMMAND", "mars") + + # Set the MARS client environment variables + + env = { + **os.environ, + "MARS_ENVIRON_ORIGIN": "polytope", + "MARS_DHS_CALLBACK_HOST": node_name, + "MARS_DHS_CALLBACK_PORT": external_port, + "MARS_DHS_LOCALPORT": container_port, + "MARS_DHS_LOCALHOST": node_name, + } + + # env = os.environ.copy() + + # def demote(user_uid, user_gid): + # def result(): + # report_ids('starting demotion') + # os.setgid(user_gid) + # os.setuid(user_uid) + # report_ids('finished demotion') + # return result + + # def report_ids(msg): + # print('uid, gid = %d, %d; %s' % (os.getuid(), os.getgid(), msg)) + + p = subprocess.Popen([mars_command, sys.argv[1]], cwd=os.path.dirname(__file__), shell=False, env=env) + return p.wait() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/aux/mars-wrapper.py b/aux/mars-wrapper.py new file mode 100644 index 0000000..781a841 --- /dev/null +++ b/aux/mars-wrapper.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. +# + + +import logging +import os +import subprocess +import sys + +import requests + +port_file = "/persistent/last_mars_port" + + +def main(): + logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(name)s %(levelname)s %(message)s") + + assert len(sys.argv) == 2 + + # Read Kubernetes service account details for authentication + with open("/var/run/secrets/kubernetes.io/serviceaccount/token", "r") as file: + token = file.read().strip() + headers = {"Authorization": "Bearer " + token} + + # Set the MARS client environment variables + node_name = os.environ["K8S_NODE_NAME"] + pod_name = os.environ["K8S_POD_NAME"] # = service name + namespace = os.environ["K8S_NAMESPACE"] + + service_url = ( + f"https://{os.environ['KUBERNETES_SERVICE_HOST']}:" + f"{os.environ['KUBERNETES_PORT_443_TCP_PORT']}/api/v1/namespaces/" + f"{namespace}/services/{pod_name}" + ) + response = requests.get( + service_url, + headers=headers, + verify="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + ) + response.raise_for_status() # Raise an exception for HTTP errors + service = response.json()["spec"] + + try: + with open(port_file, "rt") as f: + last_port_id = int(f.read()) + except FileNotFoundError: + last_port_id = 0 + + port_id = (last_port_id + 1) % 5 + + with open(port_file, "w+") as f: + f.write(str(port_id)) + + node_port = service["ports"][port_id]["nodePort"] + local_port = service["ports"][port_id]["port"] + + logging.info("Callback on {}:{}".format(node_name, node_port)) + + env = { + **os.environ, + "MARS_ENVIRON_ORIGIN": "polytope", + "MARS_DHS_CALLBACK_HOST": node_name, + "MARS_DHS_CALLBACK_PORT": str(node_port), + "MARS_DHS_LOCALPORT": str(local_port), + "MARS_DHS_LOCALHOST": pod_name, + # "MARS_DEBUG": str(1), + # "ECKIT_DEBUG": str(1), + # "FDB_DEBUG": str(1), + } + + # Call MARS + mars_command = os.environ.get("ECMWF_MARS_COMMAND", "mars") + p = subprocess.Popen([mars_command, sys.argv[1]], cwd=os.path.dirname(__file__), shell=False, env=env) + return p.wait() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/polytope_server/broker/broker.py b/polytope_server/broker/broker.py index 7af98bd..c242b1c 100644 --- a/polytope_server/broker/broker.py +++ b/polytope_server/broker/broker.py @@ -40,13 +40,10 @@ def __init__(self, config): self.collections = collection.create_collections(config.get("collections")) - self.user_limit = self.broker_config.get("user_limit", None) - def run(self): logging.info("Starting broker...") logging.info("Maximum Queue Size: {}".format(self.max_queue_size)) - logging.info("User Request Limit: {}".format(self.user_limit)) while not time.sleep(self.scheduling_interval): self.check_requests() @@ -87,43 +84,48 @@ def check_requests(self): def check_limits(self, active_requests, request): - logging.debug("Checking limits for request {}".format(request.id)) + logging.debug(f"Checking limits for request {request.id}") - # User limits - if self.user_limit is not None: - user_active_requests = sum(qr.user == request.user for qr in active_requests) - if user_active_requests >= self.user_limit: - logging.debug("User has {} of {} active requests".format(user_active_requests, self.user_limit)) - return False + # Get collection limits and calculate active requests + collection = self.collections[request.collection] + collection_limits = collection.limits + collection_total_limit = collection_limits.get("total") + collection_active_requests = sum(qr.collection == request.collection for qr in active_requests) + logging.debug(f"Collection {request.collection} has {collection_active_requests} active requests") - # Collection limits - collection_total_limit = self.collections[request.collection].limits.get("total", None) - if collection_total_limit is not None: - collection_active_requests = sum(qr.collection == request.collection for qr in active_requests) - if collection_active_requests >= collection_total_limit: + # Check collection total limit + if collection_total_limit is not None and collection_active_requests >= collection_total_limit: + logging.debug( + f"Collection has {collection_active_requests} of {collection_total_limit} total active requests" + ) + return False + + # Determine the effective limit based on role or per-user setting + role_limits = collection_limits.get("per-role", {}).get(request.user.realm, {}) + limit = max((role_limits.get(role, 0) for role in request.user.roles), default=0) + if limit == 0: # Use collection per-user limit if no role-specific limit + limit = collection_limits.get("per-user", 0) + + # Check if user exceeds the effective limit + if limit > 0: + user_active_requests = sum( + qr.collection == request.collection and qr.user == request.user for qr in active_requests + ) + if user_active_requests >= limit: logging.debug( - "Collection has {} of {} total active requests".format( - collection_active_requests, collection_total_limit - ) + f"User {request.user} has {user_active_requests} of {limit} " + f"active requests in collection {request.collection}" ) return False - - # Collection-user limits - collection_user_limit = self.collections[request.collection].limits.get("per-user", None) - if collection_user_limit is not None: - collection_user_active_requests = sum( - (qr.collection == request.collection and qr.user == request.user) for qr in active_requests - ) - if collection_user_active_requests >= collection_user_limit: + else: logging.debug( - "User has {} of {} active requests in collection {}".format( - collection_user_active_requests, - collection_user_limit, - request.collection, - ) + f"User {request.user} has {user_active_requests} of {limit} " + f"active requests in collection {request.collection}" ) - return False + return True + # Allow if no limits are exceeded + logging.debug(f"No limit for user {request.user} in collection {request.collection}") return True def enqueue(self, request): diff --git a/polytope_server/common/auth.py b/polytope_server/common/auth.py index 17374bc..8bea551 100644 --- a/polytope_server/common/auth.py +++ b/polytope_server/common/auth.py @@ -31,7 +31,6 @@ class AuthHelper: - """A helper to encapsulate checking user authentication and authorization""" def __init__(self, config): @@ -118,7 +117,7 @@ def authenticate(self, auth_header) -> User: www_authenticate=self.auth_info, ) - user.roles = ["default"] + user.roles.append("default") # Visit all authorizers to append additional roles and attributes for authorizer in self.authorizers: diff --git a/polytope_server/common/authentication/authentication.py b/polytope_server/common/authentication/authentication.py index 6602740..090c2a1 100644 --- a/polytope_server/common/authentication/authentication.py +++ b/polytope_server/common/authentication/authentication.py @@ -77,6 +77,8 @@ def name(self) -> str: "plain": "PlainAuthentication", "keycloak": "KeycloakAuthentication", "federation": "FederationAuthentication", + "jwt": "JWTAuthentication", + "openid_offline_access": "OpenIDOfflineAuthentication", } diff --git a/polytope_server/common/authentication/jwt_authentication.py b/polytope_server/common/authentication/jwt_authentication.py new file mode 100644 index 0000000..8508f0c --- /dev/null +++ b/polytope_server/common/authentication/jwt_authentication.py @@ -0,0 +1,78 @@ +# +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. +# + +import logging + +import requests +from jose import jwt + +from ..auth import User +from ..caching import cache +from ..exceptions import ForbiddenRequest +from . import authentication + + +class JWTAuthentication(authentication.Authentication): + def __init__(self, name, realm, config): + self.config = config + + self.certs_url = config["cert_url"] + self.client_id = config["client_id"] + + super().__init__(name, realm, config) + + def authentication_type(self): + return "Bearer" + + def authentication_info(self): + return "Authenticate with JWT token" + + @cache(lifetime=120) + def get_certs(self): + return requests.get(self.certs_url).json() + + @cache(lifetime=120) + def authenticate(self, credentials: str) -> User: + + try: + certs = self.get_certs() + decoded_token = jwt.decode( + token=credentials, algorithms=jwt.get_unverified_header(credentials).get("alg"), key=certs + ) + + logging.info("Decoded JWT: {}".format(decoded_token)) + + user = User(decoded_token["sub"], self.realm()) + + roles = decoded_token.get("resource_access", {}).get(self.client_id, {}).get("roles", []) + user.roles.extend(roles) + + roles = decoded_token.get("realm_access", {}).get("roles", []) + user.roles.extend(roles) + + logging.info("Found user {} from decoded JWT".format(user)) + except Exception as e: + logging.info("Failed to authenticate user from JWT") + logging.info(e) + raise ForbiddenRequest("Credentials could not be unpacked") + return user + + def collect_metric_info(self): + return {} diff --git a/polytope_server/common/authentication/mongoapikey_authentication.py b/polytope_server/common/authentication/mongoapikey_authentication.py index c5bf439..f2fcdfc 100644 --- a/polytope_server/common/authentication/mongoapikey_authentication.py +++ b/polytope_server/common/authentication/mongoapikey_authentication.py @@ -20,8 +20,7 @@ from datetime import datetime -import pymongo - +from .. import mongo_client_factory from ..auth import User from ..exceptions import ForbiddenRequest from ..metric_collector import MongoStorageMetricCollector @@ -29,7 +28,6 @@ class ApiKeyMongoAuthentication(authentication.Authentication): - """ Authenticates a user using a polytope API key. A polytope API key is an alias to a user that was previously authenticated. It allows user to authenticate once, retrieve a key, and use that for future authentication. @@ -40,19 +38,18 @@ class ApiKeyMongoAuthentication(authentication.Authentication): """ def __init__(self, name, realm, config): - self.config = config - host = config.get("host", "localhost") - port = config.get("port", "27017") + uri = config.get("uri", "mongodb://localhost:27017") collection = config.get("collection", "keys") + username = config.get("username") + password = config.get("password") - endpoint = "{}:{}".format(host, port) - self.mongo_client = pymongo.MongoClient(endpoint, journal=True, connect=False) + self.mongo_client = mongo_client_factory.create_client(uri, username, password) self.database = self.mongo_client.keys self.keys = self.database[collection] assert realm == "polytope" - self.storage_metric_collector = MongoStorageMetricCollector(endpoint, self.mongo_client, "keys", collection) + self.storage_metric_collector = MongoStorageMetricCollector(uri, self.mongo_client, "keys", collection) super().__init__(name, realm, config) @@ -63,7 +60,6 @@ def authentication_info(self): return "Authenticate with Polytope API Key from ../auth/keys" def authenticate(self, credentials: str) -> User: - # credentials should be of the form '' res = self.keys.find_one({"key.key": credentials}) if res is None: diff --git a/polytope_server/common/authentication/mongodb_authentication.py b/polytope_server/common/authentication/mongodb_authentication.py index ff1f579..a1d0be3 100644 --- a/polytope_server/common/authentication/mongodb_authentication.py +++ b/polytope_server/common/authentication/mongodb_authentication.py @@ -22,8 +22,7 @@ import binascii import hashlib -import pymongo - +from .. import mongo_client_factory from ..auth import User from ..exceptions import ForbiddenRequest from ..metric_collector import MongoStorageMetricCollector @@ -32,19 +31,18 @@ class MongoAuthentication(authentication.Authentication): def __init__(self, name, realm, config): - self.config = config - host = config.get("host", "localhost") - port = config.get("port", "27017") + uri = config.get("uri", "mongodb://localhost:27017") collection = config.get("collection", "users") + username = config.get("username") + password = config.get("password") - endpoint = "{}:{}".format(host, port) - self.mongo_client = pymongo.MongoClient(endpoint, journal=True, connect=False) + self.mongo_client = mongo_client_factory.create_client(uri, username, password) self.database = self.mongo_client.authentication self.users = self.database[collection] self.storage_metric_collector = MongoStorageMetricCollector( - endpoint, self.mongo_client, "authentication", collection + uri, self.mongo_client, "authentication", collection ) super().__init__(name, realm, config) @@ -59,7 +57,6 @@ def authentication_info(self): return "Authenticate with username and password" def authenticate(self, credentials: str) -> User: - # credentials should be of the form 'base64(:)' try: decoded = base64.b64decode(credentials).decode("utf-8") diff --git a/polytope_server/common/authentication/openid_offline_access_authentication.py b/polytope_server/common/authentication/openid_offline_access_authentication.py new file mode 100644 index 0000000..0d7dea6 --- /dev/null +++ b/polytope_server/common/authentication/openid_offline_access_authentication.py @@ -0,0 +1,113 @@ +# +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. +# + +import logging + +import requests +from jose import jwt + +from ..auth import User +from ..caching import cache +from ..exceptions import ForbiddenRequest +from . import authentication + + +class OpenIDOfflineAuthentication(authentication.Authentication): + def __init__(self, name, realm, config): + self.config = config + + self.certs_url = config["cert_url"] + self.public_client_id = config["public_client_id"] + self.private_client_id = config["private_client_id"] + self.private_client_secret = config["private_client_secret"] + self.iam_url = config["iam_url"] + self.iam_realm = config["iam_realm"] + + super().__init__(name, realm, config) + + def authentication_type(self): + return "Bearer" + + def authentication_info(self): + return "Authenticate with OpenID offline_access token" + + @cache(lifetime=120) + def get_certs(self): + return requests.get(self.certs_url).json() + + @cache(lifetime=120) + def check_offline_access_token(self, token: str) -> bool: + """ + We check if the token is recognised by the IAM service, and we cache this result. + We cannot simply try to get the access token because we would spam the IAM server with invalid tokens, and the + failure at that point would not be cached. + """ + keycloak_token_introspection = ( + self.iam_url + "/realms/" + self.iam_realm + "/protocol/openid-connect/token/introspect" + ) + introspection_data = {"token": token} + b_auth = requests.auth.HTTPBasicAuth(self.private_client_id, self.private_client_secret) + resp = requests.post(url=keycloak_token_introspection, data=introspection_data, auth=b_auth).json() + if resp["active"] and resp["token_type"] == "Offline": + return True + else: + return False + + @cache(lifetime=120) + def authenticate(self, credentials: str) -> User: + + try: + + # Check if this is a valid offline_access token + if not self.check_offline_access_token(credentials): + raise ForbiddenRequest("Not a valid offline_access token") + + # Generate an access token from the offline_access token (like a refresh token) + refresh_data = { + "client_id": self.public_client_id, + "grant_type": "refresh_token", + "refresh_token": credentials, + } + keycloak_token_endpoint = self.iam_url + "/realms/" + self.iam_realm + "/protocol/openid-connect/token" + resp = requests.post(url=keycloak_token_endpoint, data=refresh_data) + token = resp.json()["access_token"] + + certs = self.get_certs() + decoded_token = jwt.decode(token=token, algorithms=jwt.get_unverified_header(token).get("alg"), key=certs) + + logging.info("Decoded JWT: {}".format(decoded_token)) + + user = User(decoded_token["sub"], self.realm()) + + roles = decoded_token.get("resource_access", {}).get(self.public_client_id, {}).get("roles", []) + user.roles.extend(roles) + roles = decoded_token.get("realm_access", {}).get("roles", []) + user.roles.extend(roles) + + logging.info("Found user {} from openid offline_access token".format(user)) + + except Exception as e: + logging.info("Failed to authenticate user from openid offline_access token") + logging.info(e) + raise ForbiddenRequest("Could not authenticate user from openid offline_access token") + return user + + def collect_metric_info(self): + return {} diff --git a/polytope_server/common/authorization/ldap_authorization.py b/polytope_server/common/authorization/ldap_authorization.py index f062c29..6722f3b 100644 --- a/polytope_server/common/authorization/ldap_authorization.py +++ b/polytope_server/common/authorization/ldap_authorization.py @@ -19,7 +19,6 @@ # import json -import logging from ldap3 import SUBTREE, Connection, Server @@ -48,7 +47,9 @@ def get_roles(self, user: User) -> list: ) try: if self.username_attribute is None: - return retrieve_ldap_user_roles(user.username, self.filter, self.url, self.search_base, self.ldap_user, self.ldap_password) + return retrieve_ldap_user_roles( + user.username, self.filter, self.url, self.search_base, self.ldap_user, self.ldap_password + ) else: return retrieve_ldap_user_roles( user.attributes[self.username_attribute], @@ -69,10 +70,12 @@ def collect_metric_info(self): ################################################# -def retrieve_ldap_user_roles(uid: str, filter: str, url: str, search_base: str, ldap_user: str, ldap_password: str) -> list: +def retrieve_ldap_user_roles( + uid: str, filter: str, url: str, search_base: str, ldap_user: str, ldap_password: str +) -> list: """ - Takes an ECMWF UID and returns all roles matching - the provided filter 'filter'. + Takes an ECMWF UID and returns all roles matching + the provided filter 'filter'. """ server = Server(url) @@ -81,20 +84,20 @@ def retrieve_ldap_user_roles(uid: str, filter: str, url: str, search_base: str, server, user="CN={},OU=Connectors,OU=Service Accounts,DC=ecmwf,DC=int".format(ldap_user), password=ldap_password, - raise_exceptions=True + raise_exceptions=True, ) with connection as conn: conn.search( search_base=search_base, - search_filter='(&(objectClass=person)(cn={}))'.format(uid), + search_filter="(&(objectClass=person)(cn={}))".format(uid), search_scope=SUBTREE, - attributes=['memberOf'] + attributes=["memberOf"], ) user_data = json.loads(conn.response_to_json()) - if len(user_data['entries']) == 0: - raise KeyError('User {} not found in LDAP.'.format(uid)) - roles = user_data['entries'][0]['attributes']['memberOf'] + if len(user_data["entries"]) == 0: + raise KeyError("User {} not found in LDAP.".format(uid)) + roles = user_data["entries"][0]["attributes"]["memberOf"] # Filter roles matches = [] @@ -104,7 +107,7 @@ def retrieve_ldap_user_roles(uid: str, filter: str, url: str, search_base: str, # Parse CN=x,OU=y,OU=z,... into dict and extract 'common name' (CN) for i, role in enumerate(matches): - d = dict(s.split('=') for s in role.split(',')) - matches[i] = d['CN'] + d = dict(s.split("=") for s in role.split(",")) + matches[i] = d["CN"] return matches diff --git a/polytope_server/common/authorization/mongodb_authorization.py b/polytope_server/common/authorization/mongodb_authorization.py index 59fe716..48ac44c 100644 --- a/polytope_server/common/authorization/mongodb_authorization.py +++ b/polytope_server/common/authorization/mongodb_authorization.py @@ -18,8 +18,7 @@ # does it submit to any jurisdiction. # -import pymongo - +from .. import mongo_client_factory from ..auth import User from ..metric_collector import MongoStorageMetricCollector from . import authorization @@ -29,23 +28,22 @@ class MongoDBAuthorization(authorization.Authorization): def __init__(self, name, realm, config): self.config = config assert self.config["type"] == "mongodb" - self.host = config.get("host", "localhost") - self.port = config.get("port", "27017") + self.uri = config.get("uri", "mongodb://localhost:27017") self.collection = config.get("collection", "users") + username = config.get("username") + password = config.get("password") - endpoint = "{}:{}".format(self.host, self.port) - self.mongo_client = pymongo.MongoClient(endpoint, journal=True, connect=False) + self.mongo_client = mongo_client_factory.create_client(self.uri, username, password) self.database = self.mongo_client.authentication self.users = self.database[self.collection] self.storage_metric_collector = MongoStorageMetricCollector( - endpoint, self.mongo_client, "authentication", self.collection + self.uri, self.mongo_client, "authentication", self.collection ) super().__init__(name, realm, config) def get_roles(self, user: User) -> list: - if user.realm != self.realm(): raise ValueError( "Trying to authorize a user in the wrong realm, expected {}, got {}".format(self.realm(), user.realm) diff --git a/polytope_server/common/caching/caching.py b/polytope_server/common/caching/caching.py index 652ba07..9a9e91b 100644 --- a/polytope_server/common/caching/caching.py +++ b/polytope_server/common/caching/caching.py @@ -29,9 +29,9 @@ from typing import Dict, Union import pymemcache -import pymongo import redis +from .. import mongo_client_factory from ..metric import MetricType from ..metric_collector import ( DictStorageMetricCollector, @@ -195,17 +195,24 @@ def collect_metric_info(self): class MongoDBCaching(Caching): def __init__(self, cache_config): super().__init__(cache_config) - host = cache_config.get("host", "localhost") - port = cache_config.get("port", 27017) - endpoint = "{}:{}".format(host, port) + uri = cache_config.get("uri", "mongodb://localhost:27017") + + username = cache_config.get("username") + password = cache_config.get("password") + collection = cache_config.get("collection", "cache") - self.client = pymongo.MongoClient(host + ":" + str(port), journal=False, connect=False) + self.client = mongo_client_factory.create_client( + uri, + username, + password, + ) + self.database = self.client.cache self.collection = self.database[collection] self.collection.create_index("expire_at", expireAfterSeconds=0) self.collection.update_one({"_id": "hits"}, {"$setOnInsert": {"n": 0}}, upsert=True) self.collection.update_one({"_id": "misses"}, {"$setOnInsert": {"n": 0}}, upsert=True) - self.storage_metric_collector = MongoStorageMetricCollector(endpoint, self.client, "cache", collection) + self.storage_metric_collector = MongoStorageMetricCollector(uri, self.client, "cache", collection) self.cache_metric_collector = MongoCacheMetricCollector(self.client, "cache", collection) def get_type(self): @@ -220,7 +227,6 @@ def get(self, key): return obj["data"] def set(self, key, object, lifetime): - if lifetime == 0 or lifetime is None: expiry = datetime.datetime.max else: @@ -324,7 +330,6 @@ def __call__(self, f): @functools.wraps(f) def wrapper(*args, **kwargs): - cache.cancelled = False if self.cache is None: diff --git a/polytope_server/common/config/schema.yaml b/polytope_server/common/config/schema.yaml index 889c79b..9d7b162 100644 --- a/polytope_server/common/config/schema.yaml +++ b/polytope_server/common/config/schema.yaml @@ -68,7 +68,7 @@ mapping: desc: point to a hosted mongodb type: map mapping: - endpoint: + uri: desc: host and port example: localhost:27017 type: str @@ -116,7 +116,7 @@ mapping: desc: point to a hosted mongodb type: map mapping: - endpoint: + uri: desc: host and port example: localhost:27017 type: str diff --git a/polytope_server/common/datasource/datasource.py b/polytope_server/common/datasource/datasource.py index 47ea379..f5a3d80 100644 --- a/polytope_server/common/datasource/datasource.py +++ b/polytope_server/common/datasource/datasource.py @@ -19,6 +19,7 @@ # import logging +import traceback from abc import ABC from importlib import import_module from typing import Iterator @@ -80,9 +81,25 @@ def dispatch(self, request, input_data) -> bool: try: self.match(request) except Exception as e: - request.user_message += "Skipping datasource {} due to match error: {}\n".format(self.get_type(), repr(e)) + if hasattr(self, "silent_match") and self.silent_match: + pass + else: + request.user_message += "Skipping datasource {} due to match error: {}\n".format( + self.get_type(), repr(e) + ) + tb = traceback.format_exception(None, e, e.__traceback__) + logging.info(tb) + return False + # Check for datasource-specific roles + if hasattr(self, "config"): + datasource_role_rules = self.config.get("roles", None) + if datasource_role_rules is not None: + if not any(role in request.user.roles for role in datasource_role_rules.get(request.user.realm, [])): + request.user_message += "Skipping datasource {}. User is forbidden.\n".format(self.get_type()) + return False + # Retrieve/Archive/etc. success = False try: @@ -109,9 +126,11 @@ def dispatch(self, request, input_data) -> bool: "mars": "MARSDataSource", "webmars": "WebMARSDataSource", "polytope": "PolytopeDataSource", + "federated": "FederatedDataSource", "echo": "EchoDataSource", "dummy": "DummyDataSource", "raise": "RaiseDataSource", + "ionbeam": "IonBeamDataSource", } diff --git a/polytope_server/common/datasource/dummy.py b/polytope_server/common/datasource/dummy.py index 8fbfb92..682a914 100644 --- a/polytope_server/common/datasource/dummy.py +++ b/polytope_server/common/datasource/dummy.py @@ -34,27 +34,29 @@ def archive(self, request): def retrieve(self, request): try: - size = int(request.user_request.encode("utf-8")) + self.size = int(request.user_request.encode("utf-8")) except ValueError: - raise ValueError("Request should be an integer (size of random data returned)") + raise ValueError("Request should be an integer (size of random data to generate)") - if size < 0: + if self.size < 0: raise ValueError("Size must be non-negative") - repeat = b"polytope" - a, b = divmod(size, len(repeat)) - - self.data = repeat * a + repeat[:b] return True def result(self, request): - yield self.data + chunk_size = 2 * 1024 * 1024 + data_generated = 0 + while data_generated < self.size: + remaining_size = self.size - data_generated + current_chunk_size = min(chunk_size, remaining_size) + yield b"x" * current_chunk_size + data_generated += current_chunk_size def destroy(self, request) -> None: pass def mime_type(self) -> str: - return "text" + return "application/x-grib" def match(self, request): return diff --git a/polytope_server/common/datasource/fdb.py b/polytope_server/common/datasource/fdb.py index fb5c09a..503be27 100644 --- a/polytope_server/common/datasource/fdb.py +++ b/polytope_server/common/datasource/fdb.py @@ -26,7 +26,6 @@ from datetime import datetime, timedelta from pathlib import Path -import pyfdb import yaml from dateutil.relativedelta import relativedelta @@ -47,6 +46,11 @@ def __init__(self, config): self.check_schema() os.environ["FDB5_CONFIG"] = json.dumps(self.fdb_config) + os.environ["FDB_CONFIG"] = json.dumps(self.fdb_config) + os.environ["FDB5_HOME"] = self.config.get("fdb_home", "/opt/fdb") + os.environ["FDB_HOME"] = self.config.get("fdb_home", "/opt/fdb") + import pyfdb + self.fdb = pyfdb.FDB() if "spaces" in self.fdb_config: @@ -142,14 +146,25 @@ def match(self, request): r = yaml.safe_load(request.user_request) or {} for k, v in self.match_rules.items(): + + # An empty match rule means that the key must not be present + if v is None or len(v) == 0: + if k in r: + raise Exception("Request containing key '{}' is not allowed".format(k)) + else: + continue # no more checks to do + # Check that all required keys exist - if k not in r: - raise Exception("Request does not contain expected key {}".format(k)) + if k not in r and not (v is None or len(v) == 0): + raise Exception("Request does not contain expected key '{}'".format(k)) + # Process date rules if k == "date": self.date_check(r["date"], v) continue + # ... and check the value of other keys + v = [v] if isinstance(v, str) else v if r[k] not in v: raise Exception("got {} : {}, but expected one of {}".format(k, r[k], v)) diff --git a/polytope_server/common/datasource/federated.py b/polytope_server/common/datasource/federated.py new file mode 100644 index 0000000..2304b7b --- /dev/null +++ b/polytope_server/common/datasource/federated.py @@ -0,0 +1,222 @@ +# +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. +# + +import hashlib +import logging +import time +from http import HTTPStatus + +import requests + +from . import datasource + + +class FederatedDataSource(datasource.DataSource): + def __init__(self, config): + self.type = config["type"] + assert self.type == "federated" + + self.url = config["url"] + self.port = config.get("port", 443) + self.secret = config["secret"] + self.collection = config["collection"] + self.api_version = config.get("api_version", "v1") + self.result_url = None + self.mime_type_result = "application/octet-stream" + + def get_type(self): + return self.type + + def archive(self, request): + + url = "/".join( + [ + self.url + ":" + str(self.port), + "api", + self.api_version, + "requests", + self.collection, + ] + ) + logging.info("Built URL for request: {}".format(url)) + + body = { + "verb": "archive", + "request": request.user_request, + } + + headers = { + "Authorization": "Federation {}:{}:{}".format(self.secret, request.user.username, request.user.realm) + } + + # Post the initial request + + response = requests.post(url, json=body, headers=headers) + + if response.status_code != HTTPStatus.ACCEPTED: + raise Exception( + "Request could not be POSTed to remote Polytope at {}.\n\ + HTTP error code {}.\n\ + Message: {}".format( + url, response.status_code, response.content + ) + ) + + url = response.headers["location"] + + # Post the data to the upload location + + response = requests.post( + url, + self.input_data, + headers={ + **headers, + "X-Checksum": hashlib.md5(self.input_data).hexdigest(), + }, + ) + + if response.status_code != HTTPStatus.ACCEPTED: + raise Exception( + "Data could not be POSTed for upload to remote Polytope at {}.\n\ + HTTP error code {}.\n\ + Message: {}".format( + url, response.status_code, response.content + ) + ) + + url = response.headers["location"] + time.sleep(int(float(response.headers["retry-after"]))) + + status = HTTPStatus.ACCEPTED + + # Poll until the request fails or returns 200 + while status == HTTPStatus.ACCEPTED: + response = requests.get(url, headers=headers, allow_redirects=False) + status = response.status_code + logging.info(response.json()) + if "location" in response.headers: + url = response.headers["location"] + if "retry-after" in response.headers: + time.sleep(int(float(response.headers["retry-after"]))) + + if status != HTTPStatus.OK: + raise Exception( + "Request failed on remote Polytope at {}.\n\ + HTTP error code {}.\n\ + Message: {}".format( + url, status, response.json()["message"] + ) + ) + + return True + + def retrieve(self, request): + + url = "/".join( + [ + self.url + ":" + str(self.port), + "api", + self.api_version, + "requests", + self.collection, + ] + ) + logging.info("Built URL for request: {}".format(url)) + + body = { + "verb": "retrieve", + "request": request.user_request, + } + + headers = { + "Authorization": "Federation {}:{}:{}".format(self.secret, request.user.username, request.user.realm) + } + + # Post the initial request + + response = requests.post(url, json=body, headers=headers) + + if response.status_code != HTTPStatus.ACCEPTED: + raise Exception( + "Request could not be POSTed to remote Polytope at {}.\n\ + HTTP error code {}.\n\ + Message: {}".format( + url, response.status_code, response.content + ) + ) + + url = response.headers["location"] + time.sleep(int(float(response.headers["retry-after"]))) + + status = HTTPStatus.ACCEPTED + + # Poll until the request fails or returns 303 + while status == HTTPStatus.ACCEPTED: + response = requests.get(url, headers=headers, allow_redirects=False) + status = response.status_code + if "location" in response.headers: + url = response.headers["location"] + if "retry-after" in response.headers: + time.sleep(int(float(response.headers["retry-after"]))) + + if status != HTTPStatus.SEE_OTHER: + raise Exception( + "Request failed on remote Polytope at {}.\n\ + HTTP error code {}.\n\ + Message: {}".format( + url, status, response.json()["message"] + ) + ) + + self.result_url = url + + return True + + def result(self, request): + + response = requests.get(self.result_url, stream=True) + + self.mime_type_result = response.headers["Content-Type"] + + if response.status_code != HTTPStatus.OK: + raise Exception( + "Request could not be downloaded from remote Polytope at {}.\n\ + HTTP error code {}.\n\ + Message: {}".format( + self.result_url, + response.status_code, + response.json()["message"], + ) + ) + + try: + for chunk in response.iter_content(chunk_size=1024): + yield chunk + finally: + response.close() + + def mime_type(self) -> str: + return self.mime_type_result + + def destroy(self, request) -> None: + return + + def match(self, request): + return diff --git a/polytope_server/common/datasource/ionbeam.py b/polytope_server/common/datasource/ionbeam.py new file mode 100644 index 0000000..dd5b74f --- /dev/null +++ b/polytope_server/common/datasource/ionbeam.py @@ -0,0 +1,132 @@ +# +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. +# +from dataclasses import dataclass + +import requests +import yaml +from requests import Request + +from . import datasource + + +@dataclass +class IonBeamAPI: + endpoint: str + + def __post_init__(self): + assert not self.endpoint.endswith("/") + self.session = requests.Session() + + def get(self, path: str, **kwargs) -> requests.Response: + return self.session.get(f"{self.endpoint}/{path}", stream=True, **kwargs) + + def get_bytes(self, path: str, **kwargs) -> requests.Response: + kwargs["headers"] = kwargs.get("headers", {}) | {"Accept": "application/octet-stream"} + return self.get(path, **kwargs) + + def get_json(self, path, **kwargs): + return self.get(path, **kwargs).json() + + def list(self, request: dict[str, str] = {}): + return self.get_json("list", params=request) + + def head(self, request: dict[str, str] = {}): + return self.get_json("head", params=request) + + def retrieve(self, request: dict[str, str]) -> requests.Response: + return self.get_bytes("retrieve", params=request) + + def archive(self, request, file) -> requests.Response: + files = {"file": file} + return self.session.post(f"{self.endpoint}/archive", files=files, params=request) + + +class IonBeamDataSource(datasource.DataSource): + """ + Retrieve data from the IonBeam REST backend that lives here: + https://github.com/ecmwf/IonBeam-Deployment/tree/main/docker/rest_api + """ + + read_chunk_size = 2 * 1024 * 1024 + + def __init__(self, config): + """Instantiate a datasource for the IonBeam REST API""" + self.type = config["type"] + assert self.type == "ionbeam" + + self.match_rules = config.get("match", {}) + endpoint = config.get("api_endpoint", "http://iotdev-001:18201/api/v1/") + self.api = IonBeamAPI(endpoint) + + def mime_type(self) -> str: + """Returns the mimetype of the result""" + return "application/octet-stream" + + def get_type(self): + return self.type + + def archive(self, request: Request): + """Archive data, returns nothing but updates datasource state""" + r = yaml.safe_load(request.user_request) + keys = r["keys"] + + with open(r["path"], "rb") as f: + return self.api.archive(keys, f) + + def list(self, request: Request) -> list: + request_keys = yaml.safe_load(request.user_request) + return self.api.list(request_keys) + + def retrieve(self, request: Request) -> bool: + """Retrieve data, returns nothing but updates datasource state""" + + request_keys = yaml.safe_load(request.user_request) + self.response = self.api.retrieve(request_keys) + return True + + def result(self, request: Request): + """Returns a generator for the resultant data""" + return self.response.iter_content(chunk_size=self.read_chunk_size, decode_unicode=False) + + def destroy(self, request) -> None: + """A hook to do essential freeing of resources, called upon success or failure""" + + # requests response objects with stream=True can remain open indefinitely if not read to completion + # or closed explicitly + if self.response: + self.response.close() + + def match(self, request: Request) -> None: + """Checks if the request matches the datasource, raises on failure""" + + r = yaml.safe_load(request.user_request) or {} + + for k, v in self.match_rules.items(): + # Check that all required keys exist + if k not in r: + raise Exception("Request does not contain expected key {}".format(k)) + # Process date rules + if k == "date": + # self.date_check(r["date"], v) + continue + # ... and check the value of other keys + v = [v] if isinstance(v, str) else v + if r[k] not in v: + raise Exception("got {} : {}, but expected one of {}".format(k, r[k], v)) diff --git a/polytope_server/common/datasource/mars.py b/polytope_server/common/datasource/mars.py index 97600c6..ed6fd66 100644 --- a/polytope_server/common/datasource/mars.py +++ b/polytope_server/common/datasource/mars.py @@ -20,6 +20,7 @@ import logging import os +import re import tempfile from datetime import datetime, timedelta @@ -34,6 +35,7 @@ class MARSDataSource(datasource.DataSource): def __init__(self, config): assert config["type"] == "mars" + self.config = config self.type = config.get("type") self.command = config.get("command", "/usr/local/bin/mars") self.tmp_dir = config.get("tmp_dir", "/tmp") @@ -45,20 +47,38 @@ def __init__(self, config): self.subprocess = None self.fifo = None + self.silent_match = config.get("silent_match", False) + if self.match_rules is None: self.match_rules = {} self.mars_binary = config.get("binary", "mars") + self.protocol = config.get("protocol", "dhs") + + # self.fdb_config = None + self.fdb_config = config.get("fdb_config", [{}]) + if self.protocol == "remote": + # need to set FDB5 config in a /etc/fdb/config.yaml + self.fdb_home = self.tmp_dir + "/fdb-home" + # os.makedirs(self.fdb_home + "/etc/fdb/", exist_ok=True) + # with open(self.fdb_home + "/etc/fdb/config.yaml", "w") as f: + # yaml.dump(self.fdb_config, f) + # Write the mars config if "config" in config: self.mars_config = config.get("config", {}) + + if self.protocol == "remote": + self.mars_config[0]["home"] = self.fdb_home + self.mars_home = self.tmp_dir + "/mars-home" os.makedirs(self.mars_home + "/etc/mars-client/", exist_ok=True) with open(self.mars_home + "/etc/mars-client/databases.yaml", "w") as f: yaml.dump(self.mars_config, f) else: self.mars_home = None + self.mars_config = None def get_type(self): return self.type @@ -68,14 +88,31 @@ def match(self, request): r = yaml.safe_load(request.user_request) or {} for k, v in self.match_rules.items(): + + # An empty match rule means that the key must not be present + if v is None or len(v) == 0: + if k in r: + raise Exception("Request containing key '{}' is not allowed".format(k)) + else: + continue # no more checks to do + # Check that all required keys exist - if k not in r: - raise Exception("Request does not contain expected key {}".format(k)) + if k not in r and not (v is None or len(v) == 0): + raise Exception("Request does not contain expected key '{}'".format(k)) + # Process date rules if k == "date": - self.date_check(r["date"], v) + comp, v = v.split(" ", 1) + if comp == "<": + self.date_check(r["date"], v, False) + elif comp == ">": + self.date_check(r["date"], v, True) + else: + raise Exception("Invalid date comparison") continue + # ... and check the value of other keys + v = [v] if isinstance(v, str) else v if r[k] not in v: raise Exception("got {} : {}, but expected one of {}".format(k, r[k], v)) @@ -113,8 +150,10 @@ def retrieve(self, request): logging.debug("FIFO is ready for reading.") break else: + logging.debug("Detected MARS process has exited before opening FIFO.") self.destroy(request) - except Exception: + except Exception as e: + logging.error(f"Error while waiting for MARS process to open FIFO: {e}.") self.destroy(request) raise @@ -126,20 +165,22 @@ def result(self, request): for x in self.fifo.data(): yield x - self.destroy(request) + logging.info("FIFO reached EOF.") + return def destroy(self, request): try: - os.unlink(self.request_file) - except Exception: + self.subprocess.finalize(request) # Will raise if non-zero return + except Exception as e: + logging.debug("MARS subprocess failed: {}".format(e)) pass try: - self.fifo.delete() + os.unlink(self.request_file) except Exception: pass try: - self.subprocess.finalize(request, "mars -") # Will raise if non-zero return + self.fifo.delete() except Exception: pass @@ -155,32 +196,30 @@ def make_env(self, request): logging.info("Overriding MARS_USER_EMAIL with {}".format(self.override_mars_email)) mars_user = self.override_mars_email else: - mars_user = request.user.attributes["ecmwf-email"] + mars_user = request.user.attributes.get("ecmwf-email", "no-email") if self.override_mars_apikey: logging.info("Overriding MARS_USER_TOKEN with {}".format(self.override_mars_apikey)) mars_token = self.override_mars_apikey else: - mars_token = request.user.attributes["ecmwf-apikey"] + mars_token = request.user.attributes.get("ecmwf-apikey", "no-api-key") env = { **os.environ, "MARS_USER_EMAIL": mars_user, "MARS_USER_TOKEN": mars_token, - "ECMWF_MARS_COMMAND": self.mars_binary + "ECMWF_MARS_COMMAND": self.mars_binary, + "FDB5_CONFIG": yaml.dump(self.fdb_config[0]), } if self.mars_config is not None: - env = { - **os.environ, - "MARS_HOME": self.mars_home, - } + env["MARS_HOME"] = self.mars_home logging.info("Accessing MARS on behalf of user {} with token {}".format(mars_user, mars_token)) - except Exception: + except Exception as e: logging.error("MARS request aborted because user does not have associated ECMWF credentials") - raise Exception() + raise e return env @@ -195,15 +234,17 @@ def convert_to_mars_request(self, verb, user_request): request_str = request_str + "," + k + "=" + v return request_str - def check_single_date(self, date, offset, offset_fmted): + def check_single_date(self, date, offset, offset_fmted, after=False): # Date is relative (0 = now, -1 = one day ago) if str(date)[0] == "0" or str(date)[0] == "-": date_offset = int(date) dt = datetime.today() + timedelta(days=date_offset) - if dt >= offset: + if after and dt >= offset: raise Exception("Date is too recent, expected < {}".format(offset_fmted)) + elif not after and dt < offset: + raise Exception("Date is too old, expected > {}".format(offset_fmted)) else: return @@ -212,12 +253,30 @@ def check_single_date(self, date, offset, offset_fmted): dt = datetime.strptime(date, "%Y%m%d") except ValueError: raise Exception("Invalid date, expected real date in YYYYMMDD format") - if dt >= offset: + if after and dt >= offset: raise Exception("Date is too recent, expected < {}".format(offset_fmted)) + elif not after and dt < offset: + raise Exception("Date is too old, expected > {}".format(offset_fmted)) else: return - def date_check(self, date, offsets): + def parse_relativedelta(self, time_str): + + pattern = r"(\d+)([dhm])" + time_dict = {"d": 0, "h": 0, "m": 0} + matches = re.findall(pattern, time_str) + + for value, unit in matches: + if unit == "d": + time_dict["d"] += int(value) + elif unit == "h": + time_dict["h"] += int(value) + elif unit == "m": + time_dict["m"] += int(value) + + return relativedelta(days=time_dict["d"], hours=time_dict["h"], minutes=time_dict["m"]) + + def date_check(self, date, offsets, after=False): """Process special match rules for DATE constraints""" date = str(date) @@ -227,14 +286,14 @@ def date_check(self, date, offsets): date = "-1" now = datetime.today() - offset = now + relativedelta(**dict(offsets)) + offset = now - self.parse_relativedelta(offsets) offset_fmted = offset.strftime("%Y%m%d") split = str(date).split("/") # YYYYMMDD if len(split) == 1: - self.check_single_date(split[0], offset, offset_fmted) + self.check_single_date(split[0], offset, offset_fmted, after) return True # YYYYMMDD/to/YYYYMMDD -- check end and start date @@ -246,12 +305,12 @@ def date_check(self, date, offsets): if len(split) == 5 and split[3].casefold() != "by".casefold(): raise Exception("Invalid date range") - self.check_single_date(split[0], offset, offset_fmted) - self.check_single_date(split[2], offset, offset_fmted) + self.check_single_date(split[0], offset, offset_fmted, after) + self.check_single_date(split[2], offset, offset_fmted, after) return True # YYYYMMDD/YYYYMMDD/YYYYMMDD/... -- check each date for s in split: - self.check_single_date(s, offset, offset_fmted) + self.check_single_date(s, offset, offset_fmted, after) return True diff --git a/polytope_server/common/datasource/polytope.py b/polytope_server/common/datasource/polytope.py index 16200b7..66cb197 100644 --- a/polytope_server/common/datasource/polytope.py +++ b/polytope_server/common/datasource/polytope.py @@ -18,205 +18,109 @@ # does it submit to any jurisdiction. # -import hashlib +import json import logging -import time -from http import HTTPStatus +import os +import copy -import requests +import yaml +from polytope.utility.exceptions import PolytopeError +from polytope_mars.api import PolytopeMars from . import datasource class PolytopeDataSource(datasource.DataSource): def __init__(self, config): + self.config = config self.type = config["type"] assert self.type == "polytope" + self.match_rules = config.get("match", {}) + self.req_single_keys = config.get("options", {}).pop("req_single_keys", []) + self.patch_rules = config.get("patch", {}) + self.output = None - self.url = config["url"] - self.port = config.get("port", 443) - self.secret = config["secret"] - self.collection = config["collection"] - self.api_version = config.get("api_version", "v1") - self.result_url = None - self.mime_type_result = "application/octet-stream" + # Create a temp file to store gribjump config + self.config_file = "/tmp/gribjump.yaml" + with open(self.config_file, "w") as f: + f.write(yaml.dump(self.config.pop("gribjump_config"))) + self.config["datacube"]["config"] = self.config_file + os.environ["GRIBJUMP_CONFIG_FILE"] = self.config_file + + logging.info("Set up gribjump") def get_type(self): return self.type def archive(self, request): + raise NotImplementedError() - url = "/".join( - [ - self.url + ":" + str(self.port), - "api", - self.api_version, - "requests", - self.collection, - ] - ) - logging.info("Built URL for request: {}".format(url)) - - body = { - "verb": "archive", - "request": request.user_request, - } - - headers = { - "Authorization": "Federation {}:{}:{}".format(self.secret, request.user.username, request.user.realm) - } - - # Post the initial request - - response = requests.post(url, json=body, headers=headers) - - if response.status_code != HTTPStatus.ACCEPTED: - raise Exception( - "Request could not be POSTed to remote Polytope at {}.\n\ - HTTP error code {}.\n\ - Message: {}".format( - url, response.status_code, response.content - ) - ) - - url = response.headers["location"] - - # Post the data to the upload location - - response = requests.post( - url, - self.input_data, - headers={ - **headers, - "X-Checksum": hashlib.md5(self.input_data).hexdigest(), - }, - ) - - if response.status_code != HTTPStatus.ACCEPTED: - raise Exception( - "Data could not be POSTed for upload to remote Polytope at {}.\n\ - HTTP error code {}.\n\ - Message: {}".format( - url, response.status_code, response.content - ) - ) - - url = response.headers["location"] - time.sleep(int(float(response.headers["retry-after"]))) - - status = HTTPStatus.ACCEPTED - - # Poll until the request fails or returns 200 - while status == HTTPStatus.ACCEPTED: - response = requests.get(url, headers=headers, allow_redirects=False) - status = response.status_code - logging.info(response.json()) - if "location" in response.headers: - url = response.headers["location"] - if "retry-after" in response.headers: - time.sleep(int(float(response.headers["retry-after"]))) - - if status != HTTPStatus.OK: - raise Exception( - "Request failed on remote Polytope at {}.\n\ - HTTP error code {}.\n\ - Message: {}".format( - url, status, response.json()["message"] - ) - ) + def retrieve(self, request): + r = yaml.safe_load(request.user_request) + logging.info(r) - return True + # Set the "pre-path" for this request + pre_path = {} + for k,v in r.items(): + if k in self.req_single_keys: + if isinstance(v, list): + v = v[0] + pre_path[k] = v - def retrieve(self, request): + polytope_mars_config = copy.deepcopy(self.config) + polytope_mars_config["options"]["pre_path"] = pre_path - url = "/".join( - [ - self.url + ":" + str(self.port), - "api", - self.api_version, - "requests", - self.collection, - ] - ) - logging.info("Built URL for request: {}".format(url)) - - body = { - "verb": "retrieve", - "request": request.user_request, - } - - headers = { - "Authorization": "Federation {}:{}:{}".format(self.secret, request.user.username, request.user.realm) - } - - # Post the initial request - - response = requests.post(url, json=body, headers=headers) - - if response.status_code != HTTPStatus.ACCEPTED: - raise Exception( - "Request could not be POSTed to remote Polytope at {}.\n\ - HTTP error code {}.\n\ - Message: {}".format( - url, response.status_code, response.content - ) - ) - - url = response.headers["location"] - time.sleep(int(float(response.headers["retry-after"]))) - - status = HTTPStatus.ACCEPTED - - # Poll until the request fails or returns 303 - while status == HTTPStatus.ACCEPTED: - response = requests.get(url, headers=headers, allow_redirects=False) - status = response.status_code - if "location" in response.headers: - url = response.headers["location"] - if "retry-after" in response.headers: - time.sleep(int(float(response.headers["retry-after"]))) - - if status != HTTPStatus.SEE_OTHER: - raise Exception( - "Request failed on remote Polytope at {}.\n\ - HTTP error code {}.\n\ - Message: {}".format( - url, status, response.json()["message"] - ) - ) - - self.result_url = url + + polytope_mars = PolytopeMars( + polytope_mars_config, + log_context= { + "user": request.user.realm + ':' + request.user.username, + "id": request.id, + }) + + + try: + self.output = polytope_mars.extract(r) + self.output = json.dumps(self.output).encode("utf-8") + except PolytopeError as e: + raise Exception("Polytope Feature Extraction Error: {}".format(e.message)) return True def result(self, request): + logging.info("Getting result") + yield self.output - response = requests.get(self.result_url, stream=True) + def match(self, request): - self.mime_type_result = response.headers["Content-Type"] + r = yaml.safe_load(request.user_request) or {} - if response.status_code != HTTPStatus.OK: - raise Exception( - "Request could not be downloaded from remote Polytope at {}.\n\ - HTTP error code {}.\n\ - Message: {}".format( - self.result_url, - response.status_code, - response.json()["message"], - ) - ) + # Check that there is a feature specified in the request + if "feature" not in r: + raise Exception("Request does not contain expected key 'feature'") - try: - for chunk in response.iter_content(chunk_size=1024): - yield chunk - finally: - response.close() + for k, v in self.match_rules.items(): + # Check that all required keys exist + if k not in r: + raise Exception("Request does not contain expected key {}".format(k)) - def mime_type(self) -> str: - return self.mime_type_result + # ... and check the value of other keys + v = [v] if isinstance(v, str) else v + + if r[k] not in v: + raise Exception("got {} : {}, but expected one of {}".format(k, r[k], v)) + + # Check that there is only one value if required + for k, v in r.items(): + if k in self.req_single_keys: + v = [v] if isinstance(v, str) else v + if len(v) > 1: + raise Exception("Expected only one value for key {}".format(k)) + elif len(v) == 0: + raise Exception("Expected a value for key {}".format(k)) def destroy(self, request) -> None: - return + pass - def match(self, request): - return + def mime_type(self) -> str: + return "application/prs.coverage+json" diff --git a/polytope_server/common/identity/mongodb_identity.py b/polytope_server/common/identity/mongodb_identity.py index d6c68be..86a67ec 100644 --- a/polytope_server/common/identity/mongodb_identity.py +++ b/polytope_server/common/identity/mongodb_identity.py @@ -18,8 +18,7 @@ # does it submit to any jurisdiction. # -import pymongo - +from .. import mongo_client_factory from ..authentication.mongodb_authentication import MongoAuthentication from ..exceptions import Conflict, NotFound from ..metric_collector import MetricCollector, MongoStorageMetricCollector @@ -29,12 +28,17 @@ class MongoDBIdentity(identity.Identity): def __init__(self, config): self.config = config - self.host = config.get("host", "localhost") - self.port = config.get("port", "27017") + self.uri = config.get("uri", "mongodb://localhost:27017") + self.collection = config.get("collection", "users") + username = config.get("username") + password = config.get("password") - endpoint = "{}:{}".format(self.host, self.port) - self.mongo_client = pymongo.MongoClient(endpoint, journal=True, connect=False) + self.mongo_client = mongo_client_factory.create_client( + self.uri, + username, + password, + ) self.database = self.mongo_client.authentication self.users = self.database[self.collection] self.realm = config.get("realm") @@ -47,12 +51,11 @@ def __init__(self, config): pass self.storage_metric_collector = MongoStorageMetricCollector( - endpoint, self.mongo_client, "authentication", self.collection + self.uri, self.mongo_client, "authentication", self.collection ) self.identity_metric_collector = MetricCollector() def add_user(self, username: str, password: str, roles: list) -> bool: - if self.users.find_one({"username": username}) is not None: raise Conflict("Username already registered") @@ -70,7 +73,6 @@ def add_user(self, username: str, password: str, roles: list) -> bool: return True def remove_user(self, username: str) -> bool: - result = self.users.delete_one({"username": username}) if result.deleted_count > 0: return True diff --git a/polytope_server/common/io/fifo.py b/polytope_server/common/io/fifo.py index 463220e..e9f5149 100644 --- a/polytope_server/common/io/fifo.py +++ b/polytope_server/common/io/fifo.py @@ -19,13 +19,13 @@ # import errno +import logging import os import select import tempfile class FIFO: - """Creates a named pipe (FIFO) and reads data from it""" def __init__(self, name, dir=None): @@ -35,10 +35,9 @@ def __init__(self, name, dir=None): self.path = dir + "/" + name - self.delete() - os.mkfifo(self.path, 0o600) self.fifo = os.open(self.path, os.O_RDONLY | os.O_NONBLOCK) + logging.info("FIFO created") def ready(self): """Wait until FIFO is ready for reading -- i.e. opened by the writing process (man select)""" @@ -60,17 +59,20 @@ def data(self, buffer_size=2 * 1024 * 1024): if buffer != b"": yield buffer - self.delete() + # self.delete() def delete(self): """Close and delete FIFO""" + logging.info("Deleting FIFO.") try: os.close(self.fifo) - except Exception: + except Exception as e: + logging.info(f"Closing FIFO had an exception {e}") pass try: os.unlink(self.path) - except Exception: + except Exception as e: + logging.info(f"Deleting FIFO had an exception {e}") pass def read_raw(self, max_read=2 * 1024 * 1024): diff --git a/polytope_server/common/keygenerator/mongodb_keygenerator.py b/polytope_server/common/keygenerator/mongodb_keygenerator.py index 471f49b..489054e 100644 --- a/polytope_server/common/keygenerator/mongodb_keygenerator.py +++ b/polytope_server/common/keygenerator/mongodb_keygenerator.py @@ -22,8 +22,7 @@ import uuid from datetime import datetime, timedelta -import pymongo - +from .. import mongo_client_factory from ..auth import User from ..exceptions import ForbiddenRequest from ..metric_collector import MongoStorageMetricCollector @@ -34,19 +33,19 @@ class MongoKeyGenerator(keygenerator.KeyGenerator): def __init__(self, config): self.config = config assert self.config["type"] == "mongodb" - host = config.get("host", "localhost") - port = config.get("port", "27017") + uri = config.get("uri", "mongodb://localhost:27017") collection = config.get("collection", "keys") - endpoint = "{}:{}".format(host, port) - self.mongo_client = pymongo.MongoClient(endpoint, journal=True, connect=False) + username = config.get("username") + password = config.get("password") + + self.mongo_client = mongo_client_factory.create_client(uri, username, password) self.database = self.mongo_client.keys self.keys = self.database[collection] self.realms = config.get("allowed_realms") - self.storage_metric_collector = MongoStorageMetricCollector(endpoint, self.mongo_client, "keys", collection) + self.storage_metric_collector = MongoStorageMetricCollector(uri, self.mongo_client, "keys", collection) def create_key(self, user: User) -> ApiKey: - if user.realm not in self.realms: raise ForbiddenRequest("Not allowed to create an API Key for users in realm {}".format(user.realm)) diff --git a/polytope_server/common/logging.py b/polytope_server/common/logging.py index 5d1a4ed..b67c69f 100644 --- a/polytope_server/common/logging.py +++ b/polytope_server/common/logging.py @@ -21,63 +21,123 @@ import datetime import json import logging +import socket -indexable_fields = {"request_id": str} +from .. import version +# Constants for syslog facility and severity +LOCAL7 = 23 -def setup(config, source_name): - - # Override the default logger - - logger = logging.getLogger() - logger.name = source_name - handler = logging.StreamHandler() - handler.setLevel(logging.DEBUG) - - mode = config.get("logging", {}).get("mode", "json") - level = config.get("logging", {}).get("level", "INFO") +# Mapping Python logging levels to syslog severity levels +LOGGING_TO_SYSLOG_SEVERITY = { + logging.CRITICAL: 2, # LOG_CRIT + logging.ERROR: 3, # LOG_ERR + logging.WARNING: 4, # LOG_WARNING + logging.INFO: 6, # LOG_INFO + logging.DEBUG: 7, # LOG_DEBUG + logging.NOTSET: 7, # LOG_DEBUG (default) +} - handler.setFormatter(LogFormatter(mode)) - logger.addHandler(handler) - logger.setLevel(level) - - logger.info("Logging Initialized") +# Indexable fields +INDEXABLE_FIELDS = {"request_id": str} +DEFAULT_LOGGING_MODE = "json" +DEFAULT_LOGGING_LEVEL = "INFO" class LogFormatter(logging.Formatter): def __init__(self, mode): - super(LogFormatter, self).__init__() + super().__init__() self.mode = mode + def format_time(self, record): + utc_time = datetime.datetime.fromtimestamp(record.created, datetime.timezone.utc) + return utc_time.strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + + def get_hostname(self, record): + return getattr(record, "hostname", socket.gethostname()) + + def get_local_ip(self): + try: + return socket.gethostbyname(socket.gethostname()) + except Exception: + return "Unable to get IP" + + def add_indexable_fields(self, record, result): + for name, expected_type in INDEXABLE_FIELDS.items(): + if hasattr(record, name): + value = getattr(record, name) + if isinstance(value, expected_type): + result[name] = value + else: + raise TypeError(f"Extra information with key '{name}' is expected to be of type '{expected_type}'") + + def calculate_syslog_priority(self, logging_level): + severity = LOGGING_TO_SYSLOG_SEVERITY.get(logging_level, 7) # Default to LOG_DEBUG if level is not found + priority = (LOCAL7 << 3) | severity + return priority + + def format_for_logserver(self, record, result): + software_info = { + "software": "polytope-server", + "swVersion": version.__version__, + "ip": self.get_local_ip(), + } + result["origin"] = software_info + + # Ensure indexable fields are in the message + message_content = {"message": result["message"]} + for field in INDEXABLE_FIELDS: + if field in result: + message_content[field] = result[field] + result["message"] = json.dumps(message_content, indent=None) + # Add syslog facility + result["syslog_facility"] = LOCAL7 + # Add syslog severity + result["syslog_severity"] = LOGGING_TO_SYSLOG_SEVERITY.get(record.levelno, 7) + # Add syslog priority + result["syslog_priority"] = self.calculate_syslog_priority(record.levelno) + + return json.dumps(result, indent=None) + def format(self, record): + formatted_time = self.format_time(record) + result = { + "asctime": formatted_time, + "hostname": self.get_hostname(record), + "process": record.process, + "thread": record.thread, + "name": record.name, + "filename": record.filename, + "lineno": record.lineno, + "levelname": record.levelname, + "message": record.getMessage(), + } - msg = super(LogFormatter, self).format(record) + if self.mode == "console": + return f"{result['asctime']} | {result['message']}" - result = {} - result["ts"] = datetime.datetime.utcnow().isoformat()[:-3] + "Z" - result["src"] = str(record.name) - result["lvl"] = str(record.levelname) - result["pth"] = str("{}:{}".format(record.pathname, record.lineno)) - result["msg"] = str(msg) + self.add_indexable_fields(record, result) - # log accepts extra={} args to eg. logging.debug - # if the extra arguments match known indexable_fields these are added to the log - # these strongly-typed fields can be used for indexing of logs + if self.mode == "logserver": + return self.format_for_logserver(record, result) + elif self.mode == "prettyprint": + return json.dumps(result, indent=2) + else: + return json.dumps(result, indent=None) - if self.mode == "console": - return result["ts"] + " | " + result["msg"] - else: - for name, typ in indexable_fields.items(): - if hasattr(record, name): - val = getattr(record, name) - if isinstance(val, typ): - result[name] = val - else: - raise TypeError("Extra information with key {} is expected to be of type {}".format(name, typ)) - - if self.mode == "prettyprint": - indent = 2 - else: - indent = 0 - return json.dumps(result, indent=indent, sort_keys=True) +def setup(config, source_name): + logger = logging.getLogger() + logger.name = source_name + + handler = logging.StreamHandler() + handler.setLevel(logging.DEBUG) + + mode = config.get("logging", {}).get("mode", DEFAULT_LOGGING_MODE) + level = config.get("logging", {}).get("level", DEFAULT_LOGGING_LEVEL) + + handler.setFormatter(LogFormatter(mode)) + logger.addHandler(handler) + logger.setLevel(level) + + logger.info("Logging Initialized") diff --git a/polytope_server/common/metric.py b/polytope_server/common/metric.py index 9de4229..c57c48b 100644 --- a/polytope_server/common/metric.py +++ b/polytope_server/common/metric.py @@ -37,7 +37,6 @@ class MetricType(enum.Enum): class Metric: - """A sealed class representing a metric""" __slots__ = ["uuid", "timestamp", "type"] diff --git a/polytope_server/common/metric_collector/metric_collector.py b/polytope_server/common/metric_collector/metric_collector.py index 7203c44..bfea618 100644 --- a/polytope_server/common/metric_collector/metric_collector.py +++ b/polytope_server/common/metric_collector/metric_collector.py @@ -24,7 +24,6 @@ class MetricCollector: - """Interface to collect metrics from any component in Polytope for which a collector is implemented. """ diff --git a/polytope_server/common/metric_collector/queue_metric_collector.py b/polytope_server/common/metric_collector/queue_metric_collector.py index 828a774..22827fa 100644 --- a/polytope_server/common/metric_collector/queue_metric_collector.py +++ b/polytope_server/common/metric_collector/queue_metric_collector.py @@ -46,3 +46,21 @@ def total_queued(self): channel = connection.channel() q = channel.queue_declare(queue=self.queue_name, durable=True, passive=True) return q.method.message_count + + +class SQSQueueMetricCollector(QueueMetricCollector): + def __init__(self, host, client): + self.host = host + self.client = client + + def total_queued(self): + response = self.client.get_queue_attributes( + QueueUrl=self.host, + AttributeNames=[ + "ApproximateNumberOfMessages", + "ApproximateNumberOfMessagesDelayed", + "ApproximateNumberOfMessagesNotVisible", + ], + ) + values = response.get("Attributes", {}).values() + return sum(map(int, values)) diff --git a/polytope_server/common/metric_collector/storage_metric_collector.py b/polytope_server/common/metric_collector/storage_metric_collector.py index e782183..7c57ecb 100644 --- a/polytope_server/common/metric_collector/storage_metric_collector.py +++ b/polytope_server/common/metric_collector/storage_metric_collector.py @@ -112,7 +112,7 @@ def storage_space_used(self): return space_used def total_entries(self): - return self.store.count() + return self.store.count_documents({}) def db_name(self): return self.database @@ -125,10 +125,11 @@ def db_space_limit(self): class S3StorageMetricCollector(StorageMetricCollector): - def __init__(self, host, client, bucket): + def __init__(self, host, client, bucket, client_type): super().__init__(host, "s3") self.client = client self.bucket = bucket + self.client_type = client_type def collect(self): r = super().collect() @@ -141,16 +142,26 @@ def collect(self): return m def total_entries(self): - return len(list(self.client.list_objects(self.bucket))) + if self.client_type == "S3DataStaging_boto3": + # boto3 only accepts keyword arguments + return len(list(self.client.list_objects_v2(Bucket=self.bucket))) + else: + return len(list(self.client.list_objects(self.bucket))) def bucket_name(self): return self.bucket def bucket_space_used(self): size = 0 - for o in self.client.list_objects(self.bucket): - size += o.size - return size + if self.client_type == "S3DataStaging_boto3": + # boto3 only accepts keyword arguments + for o in self.client.list_objects_v2(Bucket=self.bucket)["Contents"]: + size += o["Size"] + return size + else: + for o in self.client.list_objects(self.bucket): + size += o.size + return size def bucket_space_limit(self): - return "not_implemented" + return "not implemented" diff --git a/polytope_server/common/metric_store/metric_store.py b/polytope_server/common/metric_store/metric_store.py index ecc4837..6c1bfa1 100644 --- a/polytope_server/common/metric_store/metric_store.py +++ b/polytope_server/common/metric_store/metric_store.py @@ -27,7 +27,6 @@ class MetricStore(ABC): - """MetricStore is an interface for database-based storage for Metric objects""" def __init__(self): diff --git a/polytope_server/common/metric_store/mongodb_metric_store.py b/polytope_server/common/metric_store/mongodb_metric_store.py index 9dbcdaf..b529426 100644 --- a/polytope_server/common/metric_store/mongodb_metric_store.py +++ b/polytope_server/common/metric_store/mongodb_metric_store.py @@ -22,6 +22,7 @@ import pymongo +from .. import mongo_client_factory from ..metric import ( CacheInfo, Metric, @@ -38,13 +39,13 @@ class MongoMetricStore(MetricStore): def __init__(self, config=None): - host = config.get("host", "localhost") - port = config.get("port", "27017") + uri = config.get("uri", "mongodb://localhost:27017") metric_collection = config.get("collection", "metrics") - endpoint = "{}:{}".format(host, port) + username = config.get("username") + password = config.get("password") - self.mongo_client = pymongo.MongoClient(endpoint, journal=True, connect=False) + self.mongo_client = mongo_client_factory.create_client(uri, username, password) self.database = self.mongo_client.metric_store self.store = self.database[metric_collection] @@ -58,10 +59,10 @@ def __init__(self, config=None): } self.storage_metric_collector = MongoStorageMetricCollector( - endpoint, self.mongo_client, "metric_store", metric_collection + uri, self.mongo_client, "metric_store", metric_collection ) - logging.info("MongoClient configured to open at {}".format(endpoint)) + logging.info("MongoClient configured to open at {}".format(uri)) def get_type(self): return "mongodb" @@ -85,7 +86,6 @@ def get_metric(self, uuid): return None def get_metrics(self, ascending=None, descending=None, limit=None, **kwargs): - all_slots = [] found_type = None diff --git a/polytope_server/common/mongo_client_factory.py b/polytope_server/common/mongo_client_factory.py new file mode 100644 index 0000000..7e67327 --- /dev/null +++ b/polytope_server/common/mongo_client_factory.py @@ -0,0 +1,14 @@ +import typing + +import pymongo + + +def create_client( + uri: str, + username: typing.Optional[str] = None, + password: typing.Optional[str] = None, +) -> pymongo.MongoClient: + if username and password: + return pymongo.MongoClient(host=uri, journal=True, connect=False, username=username, password=password) + else: + return pymongo.MongoClient(host=uri, journal=True, connect=False) diff --git a/polytope_server/common/queue/queue.py b/polytope_server/common/queue/queue.py index 8e8ed85..9cb1e22 100644 --- a/polytope_server/common/queue/queue.py +++ b/polytope_server/common/queue/queue.py @@ -80,7 +80,7 @@ def collect_metric_info( """Collect dictionary of metrics""" -queue_dict = {"rabbitmq": "RabbitmqQueue"} +queue_dict = {"rabbitmq": "RabbitmqQueue", "sqs": "SQSQueue"} def create_queue(queue_config): diff --git a/polytope_server/common/queue/sqs_queue.py b/polytope_server/common/queue/sqs_queue.py new file mode 100644 index 0000000..dd1a392 --- /dev/null +++ b/polytope_server/common/queue/sqs_queue.py @@ -0,0 +1,89 @@ +import json +import logging +from uuid import uuid4 + +import boto3 + +from ..metric_collector import SQSQueueMetricCollector +from . import queue + + +class SQSQueue(queue.Queue): + def __init__(self, config): + queue_name = config.get("queue_name") + region = config.get("region") + self.keep_alive_interval = config.get("keep_alive_interval", 60) + self.visibility_timeout = config.get("visibility_timeout", 120) + + logging.getLogger("sqs").setLevel(logging.WARNING) + logging.getLogger("boto3").setLevel(logging.WARNING) + logging.getLogger("botocore").setLevel(logging.WARNING) + + self.client = boto3.client("sqs", region_name=region) + + self.queue_url = self.client.get_queue_url(QueueName=queue_name).get("QueueUrl") + self.check_connection() + self.queue_metric_collector = SQSQueueMetricCollector(self.queue_url, self.client) + + def enqueue(self, message): + # Messages need to have different a `MessageGroupId` so that they can be processed in parallel. + self.client.send_message( + QueueUrl=self.queue_url, + MessageBody=json.dumps(message.body), + MessageGroupId=message.body.get("id", uuid4()), + ) + + def dequeue(self): + response = self.client.receive_message( + QueueUrl=self.queue_url, + VisibilityTimeout=self.visibility_timeout, # If processing takes more seconds, message will be read twice + MaxNumberOfMessages=1, + WaitTimeSeconds=20, + ) + if "Messages" not in response: + return None + + msg, *remainder = response["Messages"] + for item in remainder: + self.client.change_message_visibility( + QueueUrl=self.queue_url, ReceiptHandle=item["ReceiptHandle"], VisibilityTimeout=0 + ) + body = msg["Body"] + receipt_handle = msg["ReceiptHandle"] + + return queue.Message(json.loads(body), context=receipt_handle) + + def ack(self, message): + self.client.delete_message(QueueUrl=self.queue_url, ReceiptHandle=message.context) + + def nack(self, message): + self.client.change_message_visibility( + QueueUrl=self.queue_url, ReceiptHandle=message.context, VisibilityTimeout=0 + ) + + def keep_alive(self): + # Implemented for compatibility, disabled because each request to SQS is billed + pass + # return self.check_connection() + + def check_connection(self): + response = self.client.get_queue_attributes(QueueUrl=self.queue_url, AttributeNames=["CreatedTimestamp"]) + # Tries to parse response + return "Attributes" in response and "CreatedTimestamp" in response["Attributes"] + + def close_connection(self): + self.client.close() + + def count(self): + response = self.client.get_queue_attributes( + QueueUrl=self.queue_url, AttributeNames=["ApproximateNumberOfMessages"] + ) + num_messages = response["Attributes"]["ApproximateNumberOfMessages"] + + return int(num_messages) + + def get_type(self): + return "sqs" + + def collect_metric_info(self): + return self.queue_metric_collector.collect().serialize() diff --git a/polytope_server/common/request.py b/polytope_server/common/request.py index 602c63a..f2fdfb5 100644 --- a/polytope_server/common/request.py +++ b/polytope_server/common/request.py @@ -41,7 +41,6 @@ class Verb(enum.Enum): class Request: - """A sealed class representing a request""" __slots__ = [ diff --git a/polytope_server/common/request_store/mongodb_request_store.py b/polytope_server/common/request_store/mongodb_request_store.py index 1734e2d..1f43561 100644 --- a/polytope_server/common/request_store/mongodb_request_store.py +++ b/polytope_server/common/request_store/mongodb_request_store.py @@ -23,7 +23,7 @@ import pymongo -from .. import metric_store +from .. import metric_store, mongo_client_factory from ..metric import MetricType, RequestStatusChange from ..metric_collector import ( MongoRequestStoreMetricCollector, @@ -35,13 +35,12 @@ class MongoRequestStore(request_store.RequestStore): def __init__(self, config=None, metric_store_config=None): - host = config.get("host", "localhost") - port = config.get("port", "27017") + uri = config.get("uri", "mongodb://localhost:27017") request_collection = config.get("collection", "requests") + username = config.get("username") + password = config.get("password") - endpoint = "{}:{}".format(host, port) - - self.mongo_client = pymongo.MongoClient(endpoint, journal=True, connect=False) + self.mongo_client = mongo_client_factory.create_client(uri, username, password) self.database = self.mongo_client.request_store self.store = self.database[request_collection] @@ -50,11 +49,11 @@ def __init__(self, config=None, metric_store_config=None): self.metric_store = metric_store.create_metric_store(metric_store_config) self.storage_metric_collector = MongoStorageMetricCollector( - endpoint, self.mongo_client, "request_store", request_collection + uri, self.mongo_client, "request_store", request_collection ) self.request_store_metric_collector = MongoRequestStoreMetricCollector() - logging.info("MongoClient configured to open at {}".format(endpoint)) + logging.info("MongoClient configured to open at {}".format(uri)) def get_type(self): return "mongodb" @@ -87,7 +86,6 @@ def get_request(self, id): return None def get_requests(self, ascending=None, descending=None, limit=None, **kwargs): - if ascending: if ascending not in Request.__slots__: raise KeyError("Request has no key {}".format(ascending)) @@ -98,7 +96,6 @@ def get_requests(self, ascending=None, descending=None, limit=None, **kwargs): query = {} for k, v in kwargs.items(): - if k not in Request.__slots__: raise KeyError("Request has no key {}".format(k)) @@ -152,7 +149,6 @@ def update_request(self, request): return res def wipe(self): - if self.metric_store: res = self.get_requests() for i in res: diff --git a/polytope_server/common/request_store/request_store.py b/polytope_server/common/request_store/request_store.py index 4344ecc..6df5f26 100644 --- a/polytope_server/common/request_store/request_store.py +++ b/polytope_server/common/request_store/request_store.py @@ -27,7 +27,6 @@ class RequestStore(ABC): - """RequestStore is an interface for database-based storage for Request objects""" def __init__(self): diff --git a/polytope_server/common/staging/s3_boto3_staging.py b/polytope_server/common/staging/s3_boto3_staging.py new file mode 100644 index 0000000..59c483a --- /dev/null +++ b/polytope_server/common/staging/s3_boto3_staging.py @@ -0,0 +1,294 @@ +# +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. +# + +import json +import logging +import random +import time +from concurrent.futures import Future, ThreadPoolExecutor + +import boto3 +import botocore +from botocore.exceptions import ClientError + +from ..metric_collector import S3StorageMetricCollector +from . import staging + + +class AvailableThreadPoolExecutor(ThreadPoolExecutor): + def __init__(self, max_workers=None, thread_name_prefix="", initializer=None, initargs=()): + super().__init__(max_workers, thread_name_prefix, initializer, initargs) + self._running_worker_futures: set[Future] = set() + + @property + def available_workers(self) -> int: + return self._max_workers - len(self._running_worker_futures) + + def wait_for_available_worker(self, timeout=None) -> None: + start_time = time.monotonic() + while True: + if self.available_workers > 0: + return + if timeout is not None and time.monotonic() - start_time > timeout: + raise TimeoutError + time.sleep(0.1) + + def submit(self, fn, /, *args, **kwargs): + f = super().submit(fn, *args, **kwargs) + self._running_worker_futures.add(f) + f.add_done_callback(self._running_worker_futures.remove) + return f + + +class S3Staging_boto3(staging.Staging): + def __init__(self, config): + + self.bucket = config.get("bucket", "default") + self.url = config.get("url", None) + + self.host = config.get("host", "0.0.0.0") + self.port = config.get("port", "8333") + self.use_ssl = config.get("use_ssl", False) + self.max_threads = config.get("max_threads", 10) + self.buffer_size = config.get("buffer_size", 10 * 1024 * 1024) + self.should_set_policy = config.get("should_set_policy", False) + + access_key = config.get("access_key", "") + secret_key = config.get("secret_key", "") + + for name in ["boto", "urllib3", "s3transfer", "boto3", "botocore", "nose"]: + logging.getLogger(name).setLevel(logging.WARNING) + + prefix = "https" if self.use_ssl else "http" + + if config.get("random_host", False): + self.host = config.get("random_host", {}).get("host", self.host) + index = random.randint(0, config.get("random_host", {}).get("max", 1) - 1) + # replace %%ID%% in the host with the index + self.host = self.host.replace("%%ID%%", str(index)) + self.url = self.url + "/" + str(index) + logging.info(f"Using random host: {self.host}") + + self._internal_url = f"{prefix}://{self.host}:{self.port}" + + # Setup Boto3 client + self.s3_client = boto3.client( + "s3", + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + endpoint_url=self._internal_url, + config=botocore.config.Config( + max_pool_connections=50, + s3={"addressing_style ": "path"}, + ), + # use_ssl=self.use_ssl, + ) + + # Attempt to create the bucket + try: + self.s3_client.create_bucket(Bucket=self.bucket) + except self.s3_client.exceptions.BucketAlreadyExists: + logging.info(f"Bucket {self.bucket} already exists.") + except self.s3_client.exceptions.BucketAlreadyOwnedByYou: + logging.info(f"Bucket {self.bucket} already exists and owned by you.") + except ClientError as e: + logging.error(f"Error creating bucket: {e}") + # Set bucket policy + if self.should_set_policy: + self.set_bucket_policy() + self.storage_metric_collector = S3StorageMetricCollector( + self.host, self.s3_client, self.bucket, self.get_type() + ) + + logging.info(f"Opened data staging at {self.host}:{self.port} with bucket {self.bucket}") + + def create(self, name, data, content_type): + name = name + ".grib" + # fix for seaweedfs auto-setting Content-Disposition to inline and earthkit expecting extension, + # else using content-disposition header + try: + multipart_upload = self.s3_client.create_multipart_upload( + Bucket=self.bucket, Key=name, ContentType=content_type, ContentDisposition="attachment" + ) + upload_id = multipart_upload["UploadId"] + + parts = [] + part_number = 1 + futures = [] + + with AvailableThreadPoolExecutor(max_workers=self.max_threads) as executor: + executor.wait_for_available_worker() + if not data: + logging.info(f"No data provided. Uploading a single empty part for {name}.") + else: + for part_data in self.iterator_buffer(data, self.buffer_size): + if part_data: + futures.append(executor.submit(self.upload_part, name, part_number, part_data, upload_id)) + part_number += 1 + + for future in futures: + result = future.result() + parts.append(result) + + if not parts: + logging.warning(f"No parts uploaded for {name}. Aborting upload.") + self.s3_client.abort_multipart_upload(Bucket=self.bucket, Key=name, UploadId=upload_id) + raise ValueError("No data retrieved") + + self.s3_client.complete_multipart_upload( + Bucket=self.bucket, Key=name, UploadId=upload_id, MultipartUpload={"Parts": parts} + ) + + logging.info(f"Successfully uploaded {name} in {len(parts)} parts.") + return self.get_url(name) + + except ClientError as e: + logging.error(f"Failed to upload {name}: {e}") + if "upload_id" in locals(): + self.s3_client.abort_multipart_upload(Bucket=self.bucket, Key=name, UploadId=upload_id) + raise + + def upload_part(self, name, part_number, data, upload_id): + logging.debug(f"Uploading part {part_number} of {name}, {len(data)} bytes") + response = self.s3_client.upload_part( + Bucket=self.bucket, Key=name, PartNumber=part_number, UploadId=upload_id, Body=data + ) + return {"PartNumber": part_number, "ETag": response["ETag"]} + + def set_bucket_policy(self): + """ + Grants read access to individual objects - user has access to all objects, but would need to know the UUID. + Denies read access to the bucket (cannot list objects) - important, so users cannot see all UUIDs! + Denies read access to the bucket location + """ + policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowReadAccessToIndividualObjects", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": f"arn:aws:s3:::{self.bucket}/*", + }, + { + "Sid": "AllowListBucket", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:ListBucket", + "Resource": f"arn:aws:s3:::{self.bucket}", + }, + { + "Sid": "AllowGetBucketLocation", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetBucketLocation", + "Resource": f"arn:aws:s3:::{self.bucket}", + }, + ], + } + self.s3_client.put_bucket_policy(Bucket=self.bucket, Policy=json.dumps(policy)) + + def read(self, name): + try: + response = self.s3_client.get_object(Bucket=self.bucket, Key=name) + return response["Body"].read() + except ClientError as e: + logging.error(f"Could not read object {name}: {e}") + raise KeyError(name) + + def delete(self, name): + try: + self.s3_client.delete_object(Bucket=self.bucket, Key=name) + return True + except ClientError as e: + logging.error(f"Could not delete object {name}: {e}") + raise KeyError(name) + + def query(self, name): + try: + self.s3_client.head_object(Bucket=self.bucket, Key=name) + return True + except ClientError: + return False + + def stat(self, name): + try: + response = self.s3_client.head_object(Bucket=self.bucket, Key=name) + return response["ContentType"], response["ContentLength"] + except ClientError as e: + logging.error(f"Could not stat object {name}: {e}") + raise KeyError(name) + + def get_url(self, name): + if self.url: + return f"{self.url}/{self.bucket}/{name}" + return None + + def get_internal_url(self, name): + pass + + def get_type(self): + return "S3DataStaging_boto3" + + def list(self): + try: + resources = [] + data = self.s3_client.list_objects_v2(Bucket=self.bucket, MaxKeys=999999999999999) + + if data.get("contents", {}).get("IsTruncated ncated", False): + logging.warning("Truncated list of objects. Some objects may not be listed.") + + if "Contents" not in data: # No objects in the bucket + return resources + for o in data["Contents"]: + resources.append(staging.ResourceInfo(o["Key"], o["Size"])) + return resources + except ClientError as e: + logging.error(f"Failed to list objects: {e}") + raise + + def wipe(self): + objects_to_delete = self.list() + delete_objects = [{"Key": obj} for obj in objects_to_delete] + if delete_objects: + try: + logging.info(f"Deleting {len(delete_objects)} : {delete_objects} objects from {self.bucket}") + self.s3_client.delete_objects(Bucket=self.bucket, Delete={"Objects": delete_objects}) + except ClientError as e: + logging.error(f"Error deleting objects: {e}") + raise + + def collect_metric_info(self): + return self.storage_metric_collector.collect().serialize() + + def get_url_prefix(self): + return "{}/".format(self.bucket) + + def iterator_buffer(self, iterable, buffer_size): + buffer = b"" + for data in iterable: + buffer += data + if len(buffer) >= buffer_size: + output, leftover = buffer[:buffer_size], buffer[buffer_size:] + buffer = leftover + yield output + + yield buffer diff --git a/polytope_server/common/staging/s3_staging.py b/polytope_server/common/staging/s3_staging.py index bac873c..6f11e44 100644 --- a/polytope_server/common/staging/s3_staging.py +++ b/polytope_server/common/staging/s3_staging.py @@ -27,49 +27,99 @@ # ####################################################################### +import copy import json import logging +import time +from concurrent.futures import Future, ThreadPoolExecutor import minio from minio import Minio from minio.definitions import UploadPart -from minio.error import BucketAlreadyOwnedByYou, NoSuchKey +from minio.error import BucketAlreadyExists, BucketAlreadyOwnedByYou, NoSuchKey from ..metric_collector import S3StorageMetricCollector from . import staging +class AvailableThreadPoolExecutor(ThreadPoolExecutor): + + def __init__(self, max_workers=None, thread_name_prefix="", initializer=None, initargs=()): + super().__init__(max_workers, thread_name_prefix, initializer, initargs) + self._running_worker_futures: set[Future] = set() + + @property + def available_workers(self) -> int: + return self._max_workers - len(self._running_worker_futures) + + def wait_for_available_worker(self, timeout=None) -> None: + start_time = time.monotonic() + while True: + if self.available_workers > 0: + return + if timeout is not None and time.monotonic() - start_time > timeout: + raise TimeoutError + time.sleep(0.1) + + def submit(self, fn, /, *args, **kwargs): + f = super().submit(fn, *args, **kwargs) + self._running_worker_futures.add(f) + f.add_done_callback(self._running_worker_futures.remove) + return f + + class S3Staging(staging.Staging): def __init__(self, config): self.host = config.get("host", "0.0.0.0") self.port = config.get("port", "8000") + self.max_threads = config.get("max_threads", 20) + self.buffer_size = config.get("buffer_size", 20 * 1024 * 1024) endpoint = "{}:{}".format(self.host, self.port) access_key = config.get("access_key", "") secret_key = config.get("secret_key", "") self.bucket = config.get("bucket", "default") + secure = config.get("secure", False) self.url = config.get("url", None) internal_url = "{}:{}".format(self.host, self.port) - self.client = Minio( - internal_url, - access_key=access_key, - secret_key=secret_key, - secure=False, - ) - self.internal_url = "http://" + internal_url + secure = config.get("use_ssl", False) + + if access_key == "" or secret_key == "": + self.client = Minio( + internal_url, + secure=secure, + ) + + else: + self.client = Minio( + internal_url, + access_key=access_key, + secret_key=secret_key, + secure=secure, + ) + + self.internal_url = ("https://" if secure else "http://") + internal_url try: self.client.make_bucket(self.bucket) + self.client.set_bucket_policy(self.bucket, self.bucket_policy()) + except BucketAlreadyExists: + pass except BucketAlreadyOwnedByYou: pass - self.client.set_bucket_policy(self.bucket, self.bucket_policy()) - - self.storage_metric_collector = S3StorageMetricCollector(endpoint, self.client, self.bucket) + self.storage_metric_collector = S3StorageMetricCollector(endpoint, self.client, self.bucket, self.get_type()) logging.info( "Opened data staging at {}:{}/{}, locatable from {}".format(self.host, self.port, self.bucket, self.url) ) + def upload_part(self, part_number, buf, metadata, name, upload_id): + logging.debug(f"Uploading part {part_number} ({len(buf)} bytes) of {name}") + etag = self.client._do_put_object( + self.bucket, name, buf, len(buf), part_number=part_number, metadata=metadata, upload_id=upload_id + ) + return etag, len(buf) + def create(self, name, data, content_type): url = self.get_url(name) logging.info("Putting to staging: {}".format(name)) @@ -79,52 +129,39 @@ def create(self, name, data, content_type): upload_id = self.client._new_multipart_upload(self.bucket, name, metadata) - i = 1 parts = {} - total_size = 0 - for buf in self.iterator_buffer(data, 200 * 1024 * 1024): - if len(buf) == 0: - break - try: - logging.info("Uploading part {} ({} bytes) of {}".format(i, len(buf), name)) - etag = self.client._do_put_object( - self.bucket, - name, - buf, - len(buf), - part_number=i, - metadata=metadata, - upload_id=upload_id, - ) - total_size += len(buf) - parts[i] = UploadPart(self.bucket, name, upload_id, i, etag, None, len(buf)) - i += 1 - except Exception: - self.client._remove_incomplete_upload(self.bucket, name, upload_id) - raise + part_number = 1 + futures = [] + + with AvailableThreadPoolExecutor(max_workers=self.max_threads) as executor: + executor.wait_for_available_worker() + for buf in self.iterator_buffer(data, self.buffer_size): + if len(buf) == 0: + break + future = executor.submit(self.upload_part, copy.copy(part_number), buf, metadata, name, upload_id) + futures.append((future, part_number)) + part_number += 1 - if len(parts) == 0: + try: + for future, part_number in futures: + etag, size = future.result() + parts[part_number] = UploadPart(self.bucket, name, upload_id, part_number, etag, None, size) + except Exception as e: + logging.error(f"Error uploading parts: {str(e)}") + self.client._remove_incomplete_upload(self.bucket, name, upload_id) + raise + + # Completing upload + try: + logging.info(parts) try: - logging.info("Uploading single empty part of {}".format(name)) - etag = self.client._do_put_object( - self.bucket, - name, - b"", - 0, - part_number=i, - metadata=metadata, - upload_id=upload_id, - ) - total_size += 0 - parts[i] = UploadPart(self.bucket, name, upload_id, i, etag, None, 0) - i += 1 + self.client._complete_multipart_upload(self.bucket, name, upload_id, parts) except Exception: - self.client._remove_incomplete_upload(self.bucket, name, upload_id) - raise + time.sleep(5) + self.client._complete_multipart_upload(self.bucket, name, upload_id, parts) - try: - self.client._complete_multipart_upload(self.bucket, name, upload_id, parts) - except Exception: + except Exception as e: + logging.error(f"Error completing multipart upload: {str(e)}") self.client._remove_incomplete_upload(self.bucket, name, upload_id) raise @@ -201,25 +238,25 @@ def bucket_policy(self): "Version": "2012-10-17", "Statement": [ { - "Sid": "", - "Effect": "Deny", - "Principal": {"AWS": "*"}, - "Action": "s3:GetBucketLocation", - "Resource": "arn:aws:s3:::{}".format(self.bucket), + "Sid": "AllowReadAccessToIndividualObjects", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": f"arn:aws:s3:::{self.bucket}/*", }, { - "Sid": "", - "Effect": "Deny", - "Principal": {"AWS": "*"}, + "Sid": "AllowListBucket", + "Effect": "Allow", + "Principal": "*", "Action": "s3:ListBucket", - "Resource": "arn:aws:s3:::{}".format(self.bucket), + "Resource": f"arn:aws:s3:::{self.bucket}", }, { - "Sid": "", + "Sid": "AllowGetBucketLocation", "Effect": "Allow", - "Principal": {"AWS": "*"}, - "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::{}/*".format(self.bucket), + "Principal": "*", + "Action": "s3:GetBucketLocation", + "Resource": f"arn:aws:s3:::{self.bucket}", }, ], } diff --git a/polytope_server/common/staging/staging.py b/polytope_server/common/staging/staging.py index 3a0b2ab..f18de89 100644 --- a/polytope_server/common/staging/staging.py +++ b/polytope_server/common/staging/staging.py @@ -25,7 +25,11 @@ from ..metric import MetricType from ..request import Status -type_to_class_map = {"polytope": "PolytopeStaging", "s3": "S3Staging"} +type_to_class_map = { + "polytope": "PolytopeStaging", + "s3": "S3Staging", + "s3_boto3": "S3Staging_boto3", +} class ResourceInfo: @@ -33,6 +37,9 @@ def __init__(self, name, size): self.name = name self.size = size + def __repr__(self): + return f"ResourceInfo({self.name}, {self.size})" + class Staging(ABC): def __init__(self, staging_config=None): diff --git a/polytope_server/common/subprocess.py b/polytope_server/common/subprocess.py index 5e010df..fd01575 100644 --- a/polytope_server/common/subprocess.py +++ b/polytope_server/common/subprocess.py @@ -46,7 +46,7 @@ def running(self): def returncode(self): return self.subprocess.poll() - def finalize(self, request, filter): + def finalize(self, request, filter=None): """Close subprocess and decode output""" out, _ = self.subprocess.communicate() @@ -54,7 +54,7 @@ def finalize(self, request, filter): self.output = out.decode().splitlines() for line in self.output: - if filter in line: + if filter and filter in line: request.user_message += line + "\n" if self.returncode() != 0: diff --git a/polytope_server/common/user.py b/polytope_server/common/user.py index 9f522cc..c9a2d61 100644 --- a/polytope_server/common/user.py +++ b/polytope_server/common/user.py @@ -68,3 +68,6 @@ def serialize(self): v = self.__getattribute__(k) result[k] = v return result + + def __str__(self): + return f"User({self.realm}:{self.username})" diff --git a/polytope_server/frontend/common/application_server.py b/polytope_server/frontend/common/application_server.py index b3f63c8..8083c4b 100644 --- a/polytope_server/frontend/common/application_server.py +++ b/polytope_server/frontend/common/application_server.py @@ -22,7 +22,6 @@ import gunicorn.app.base from gunicorn import glogging -from gunicorn.six import iteritems class CustomLogger(glogging.Logger): @@ -46,9 +45,9 @@ def __init__(self, app, options=None): def load_config(self): config = dict( - [(key, value) for key, value in iteritems(self.options) if key in self.cfg.settings and value is not None] + [(key, value) for key, value in self.options.items() if key in self.cfg.settings and value is not None] ) - for key, value in iteritems(config): + for key, value in config.items(): self.cfg.set(key.lower(), value) self.cfg.set("logger_class", CustomLogger) diff --git a/polytope_server/frontend/common/data_transfer.py b/polytope_server/frontend/common/data_transfer.py index a87974b..e0b186d 100644 --- a/polytope_server/frontend/common/data_transfer.py +++ b/polytope_server/frontend/common/data_transfer.py @@ -131,9 +131,20 @@ def upload(self, id, http_request): def process_download(self, request): try: - request.content_type, request.content_length = self.staging.stat(request.id) + + # TODO: temporary fix for Content-Disposition earthkit issues + split = request.url.split("/")[-1].split(".") + extension = None + if len(split) > 1: + extension = split[-1] + + object_id = request.id + if extension is not None: + object_id = request.id + "." + extension + + request.content_type, request.content_length = self.staging.stat(object_id) except Exception: - raise ServerError("Error while querying data staging with {}".format(request.id)) + raise ServerError("Error while querying data staging with {}".format(object_id)) response = self.construct_response(request) return RequestRedirected(response) diff --git a/polytope_server/frontend/common/flask_decorators.py b/polytope_server/frontend/common/flask_decorators.py index 2a5280e..f94d68f 100644 --- a/polytope_server/frontend/common/flask_decorators.py +++ b/polytope_server/frontend/common/flask_decorators.py @@ -18,7 +18,7 @@ # does it submit to any jurisdiction. # -import collections +import collections.abc import json from flask import Response @@ -31,13 +31,13 @@ def RequestSucceeded(response): - if not isinstance(response, collections.Mapping): + if not isinstance(response, collections.abc.Mapping): response = {"message": response} return Response(response=json.dumps(response), status=200, mimetype="application/json") def RequestAccepted(response): - if not isinstance(response, collections.Mapping): + if not isinstance(response, collections.abc.Mapping): response = {"message": response} if response["message"] == "": response["message"] = "Request {}".format(response["status"]) diff --git a/polytope_server/frontend/flask_handler.py b/polytope_server/frontend/flask_handler.py index 86f2024..303f793 100644 --- a/polytope_server/frontend/flask_handler.py +++ b/polytope_server/frontend/flask_handler.py @@ -20,7 +20,6 @@ import json import logging -import os import pathlib import tempfile @@ -29,6 +28,7 @@ from flask import Flask, request from flask_swagger_ui import get_swaggerui_blueprint from werkzeug.exceptions import default_exceptions +from werkzeug.middleware.proxy_fix import ProxyFix from ..common.exceptions import BadRequest, ForbiddenRequest, HTTPException, NotFound from ..version import __version__ @@ -47,9 +47,13 @@ def create_handler( collections, identity, apikeygenerator, + proxy_support: bool, ): handler = Flask(__name__) + if proxy_support: + handler.wsgi_app = ProxyFix(handler.wsgi_app, x_for=1, x_proto=1, x_host=1) + openapi_spec = "static/openapi.yaml" spec_path = pathlib.Path(__file__).parent.absolute() / openapi_spec with spec_path.open("r+", encoding="utf8") as f: @@ -62,7 +66,8 @@ def create_handler( SWAGGERUI_BLUEPRINT = get_swaggerui_blueprint( SWAGGER_URL, tmp.name, config={"app_name": "Polytope", "spec": spec} ) - handler.register_blueprint(SWAGGERUI_BLUEPRINT, url_prefix=SWAGGER_URL) + handler.register_blueprint(SWAGGERUI_BLUEPRINT, name="openapi", url_prefix=SWAGGER_URL) + handler.register_blueprint(SWAGGERUI_BLUEPRINT, name="home", url_prefix="/") data_transfer = DataTransfer(request_store, staging) @@ -87,13 +92,6 @@ def handle_error(error): for code, ex in default_exceptions.items(): handler.errorhandler(code)(handle_error) - @handler.route("/", methods=["GET"]) - def root(): - this_dir = os.path.dirname(os.path.abspath(__file__)) + "/" - with open(this_dir + "web/index.html") as fh: - content = fh.read() - return content - def get_auth_header(request): return request.headers.get("Authorization", "") @@ -227,6 +225,19 @@ def list_collections(): pass return RequestSucceeded(authorized_collections) + # New handler + # @handler.route("/api/v1/collection/", methods=["GET"]) + # def describe_collection(collection): + # auth_header = get_auth_header(request) + # authorized_collections = [] + # for name, collection in collections.items(): + # try: + # if auth.can_access_collection(auth_header, collection): + # authorized_collections.append(name) + # except ForbiddenRequest: + # pass + # return RequestSucceeded(authorized_collections) + @handler.after_request def add_header(response: flask.Response): response.cache_control.no_cache = True @@ -249,7 +260,6 @@ def only_json(): return handler def run_server(self, handler, server_type, host, port): - if server_type == "flask": # flask internal server for non-production environments # should only be used for testing and debugging diff --git a/polytope_server/frontend/frontend.py b/polytope_server/frontend/frontend.py index 2c4d7e1..6481118 100644 --- a/polytope_server/frontend/frontend.py +++ b/polytope_server/frontend/frontend.py @@ -59,7 +59,6 @@ def __init__(self, config): self.port = frontend_config.get("port", "5000") def run(self): - # create instances of authentication, request_store & staging request_store = create_request_store(self.config.get("request_store"), self.config.get("metric_store")) @@ -72,7 +71,15 @@ def run(self): handler_module = importlib.import_module("polytope_server.frontend." + self.handler_type + "_handler") handler_class = getattr(handler_module, self.handler_dict[self.handler_type])() - handler = handler_class.create_handler(request_store, auth, staging, collections, identity, apikeygenerator) + handler = handler_class.create_handler( + request_store, + auth, + staging, + collections, + identity, + apikeygenerator, + self.config.get("frontend", {}).get("proxy_support", False), + ) logging.info("Starting frontend...") handler_class.run_server(handler, self.server_type, self.host, self.port) diff --git a/polytope_server/frontend/web/css/style.css b/polytope_server/frontend/web/css/style.css deleted file mode 100644 index 5629028..0000000 --- a/polytope_server/frontend/web/css/style.css +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -In applying this licence, ECMWF does not waive the privileges and immunities -granted to it by virtue of its status as an intergovernmental organisation nor -does it submit to any jurisdiction. -*/ - -.wide { - text-align: center; -} - -.wide h1 { - display: inline-block; -} diff --git a/polytope_server/frontend/web/index.html b/polytope_server/frontend/web/index.html deleted file mode 100644 index b578842..0000000 --- a/polytope_server/frontend/web/index.html +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - Polytope - - - -
-

Polytope

-

- This is an experimental service for EU projects, see - https://github.com/ecmwf-projects/polytope-server. -

-
- - diff --git a/polytope_server/garbage-collector/garbage_collector.py b/polytope_server/garbage-collector/garbage_collector.py index 822ff7b..5e0a131 100644 --- a/polytope_server/garbage-collector/garbage_collector.py +++ b/polytope_server/garbage-collector/garbage_collector.py @@ -21,7 +21,7 @@ import logging import re import time -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from ..common.request import Status from ..common.request_store import create_request_store @@ -63,7 +63,7 @@ def run(self): def remove_old_requests(self): """Removes requests that are FAILED or PROCESSED after the configured time""" - now = datetime.utcnow() + now = datetime.now(timezone.utc) cutoff = now - self.age requests = self.request_store.get_requests(status=Status.FAILED) + self.request_store.get_requests( @@ -71,22 +71,37 @@ def remove_old_requests(self): ) for r in requests: - if datetime.fromtimestamp(r.last_modified) < cutoff: + data_name = r.id + ".grib" # TODO temporary fix + if datetime.fromtimestamp(r.last_modified, tz=timezone.utc) < cutoff: logging.info("Deleting {} because it is too old.".format(r.id)) try: - self.staging.delete(r.id) + self.staging.delete(data_name) except KeyError: - pass + logging.info(f"Removing old request but data {data_name} not found in staging.") self.request_store.remove_request(r.id) def remove_dangling_data(self): """As a failsafe, removes data which has no corresponding request.""" all_objects = self.staging.list() for data in all_objects: - request = self.request_store.get_request(id=data.name) + + # logging.info(f"Checking {data.name}") + + # TODO: fix properly + # remove file extension if it exists + request_id = data.name + if "." in request_id: + request_id = request_id.split(".")[0] + + request = self.request_store.get_request(id=request_id) + if request is None: - logging.info("Deleting {} because it has no matching request.".format(data.name)) - self.staging.delete(data.name) + logging.info("Deleting {} because it has no matching request.".format(request_id)) + try: + self.staging.delete(data.name) # TODO temporary fix for content-disposition error + except KeyError: + # TODO: why does this happen? + logging.info("Data {} not found in staging.".format(data.name)) def remove_by_size(self): """Cleans data according to size limits of the staging, removing older requests first.""" @@ -100,8 +115,8 @@ def remove_by_size(self): logging.info( "Found {} items in staging -- {}/{} bytes -- {:3.1f}%".format( len(all_objects), - total_size, - self.threshold, + format_bytes(total_size), + format_bytes(self.threshold), total_size / self.threshold * 100, ) ) @@ -109,7 +124,18 @@ def remove_by_size(self): all_objects_by_age = {} for data in all_objects: + + # TODO: fix properly + # remove file extension if it exists + if "." in data.name: + data.name = data.name.split(".")[0] + request = self.request_store.get_request(id=data.name) + + if request is None: + logging.info(f"Skipping request {data.name}, not found in request store.") + continue + all_objects_by_age[data.name] = { "size": data.size, "last_modified": request.last_modified, @@ -122,10 +148,18 @@ def remove_by_size(self): # Delete objects in ascending last_modified order (oldest first) for name, v in sorted(all_objects_by_age.items(), key=lambda x: x[1]["last_modified"]): logging.info("Deleting {} because threshold reached and it is the oldest request.".format(name)) - self.staging.delete(name) + try: + self.staging.delete(name) + except KeyError: + logging.info("Data {} not found in staging.".format(name)) + + # TODO: fix properly + if "." in name: + name = name.split(".")[0] + self.request_store.remove_request(name) total_size -= v["size"] - logging.info("Size of staging is {}/{}".format(total_size, self.threshold)) + logging.info("Size of staging is {}/{}".format(format_bytes(total_size), format_bytes(self.threshold))) if total_size < self.threshold: return @@ -161,3 +195,14 @@ def parse_bytes(size_str): return size * 1024**4 return False + + +def format_bytes(size_bytes): + if size_bytes == 0: + return "0B" + size_names = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB") + i = 0 + while size_bytes >= 1024 and i < len(size_names) - 1: + size_bytes /= 1024.0 + i += 1 + return "{:.1f} {}".format(size_bytes, size_names[i]) diff --git a/polytope_server/version.py b/polytope_server/version.py index 2290349..aefc76a 100644 --- a/polytope_server/version.py +++ b/polytope_server/version.py @@ -20,4 +20,4 @@ # Single-source Polytope version number -__version__ = "0.7.8" +__version__ = "0.8.0" diff --git a/polytope_server/worker/worker.py b/polytope_server/worker/worker.py index 5bfac20..9a21d50 100644 --- a/polytope_server/worker/worker.py +++ b/polytope_server/worker/worker.py @@ -23,6 +23,7 @@ import signal import sys import time +import traceback from concurrent.futures import ThreadPoolExecutor import requests @@ -30,7 +31,7 @@ from ..common import collection, metric_store from ..common import queue as polytope_queue from ..common import request_store, staging -from ..common.metric import MetricType, WorkerInfo, WorkerStatusChange +from ..common.metric import WorkerInfo, WorkerStatusChange from ..common.request import Status @@ -84,13 +85,6 @@ def __init__(self, config, debug=False, is_webmars=False): signal.signal(signal.SIGINT, self.on_process_terminated) signal.signal(signal.SIGTERM, self.on_process_terminated) - def __del__(self): - if self.metric_store: - self.metric_store.remove_metric(self.metric.uuid) - res = self.metric_store.get_metrics(type=MetricType.WORKER_STATUS_CHANGE, host=self.metric.host) - for i in res: - self.metric_store.remove_metric(i.uuid) - def update_status(self, new_status, time_spent=None, request_id=None): if time_spent is None: time_spent = self.poll_interval @@ -117,6 +111,7 @@ def update_status(self, new_status, time_spent=None, request_id=None): "Worker status update", extra=WorkerStatusChange(status=self.status).serialize(), ) + self.update_metric() def update_metric(self): self.metric.update( @@ -133,81 +128,85 @@ def update_metric(self): def run(self): - self.queue = polytope_queue.create_queue(self.config.get("queue")) - self.thread_pool = ThreadPoolExecutor(1) - self.update_status("idle", time_spent=0) - self.update_metric() - - while not time.sleep(self.poll_interval): - - self.queue.keep_alive() - - # No active request: try to pop from queue and process request in future thread - if self.future is None: - self.queue_msg = self.queue.dequeue() - if self.queue_msg is not None: - id = self.queue_msg.body["id"] - self.request = self.request_store.get_request(id) - - # This occurs when a request has been revoked while it was on the queue - if self.request is None: - logging.info( - "Request no longer exists, ignoring", - extra={"request_id": id}, - ) - self.update_status("idle") - self.queue.ack(self.queue_msg) - - # Occurs if a request crashed a worker and the message gets requeued (status will be PROCESSING) - # We do not want to try this request again - elif self.request.status != Status.QUEUED: - logging.info( - "Request has unexpected status {}, setting to failed".format(self.request.status), - extra={"request_id": id}, - ) - self.request.set_status(Status.FAILED) - self.request_store.update_request(self.request) + try: + self.queue = polytope_queue.create_queue(self.config.get("queue")) + + self.update_status("idle", time_spent=0) + # self.update_metric() + + while not time.sleep(self.poll_interval): + self.queue.keep_alive() + + # No active request: try to pop from queue and process request in future thread + if self.future is None: + self.queue_msg = self.queue.dequeue() + if self.queue_msg is not None: + id = self.queue_msg.body["id"] + self.request = self.request_store.get_request(id) + + # This occurs when a request has been revoked while it was on the queue + if self.request is None: + logging.info( + "Request no longer exists, ignoring", + extra={"request_id": id}, + ) + self.update_status("idle") + self.queue.ack(self.queue_msg) + + # Occurs if a request crashed a worker and the message gets requeued (status will be PROCESSING) + # We do not want to try this request again + elif self.request.status != Status.QUEUED: + logging.info( + "Request has unexpected status {}, setting to failed".format(self.request.status), + extra={"request_id": id}, + ) + self.request.set_status(Status.FAILED) + self.request_store.update_request(self.request) + self.update_status("idle") + self.queue.ack(self.queue_msg) + + # OK, process the request + else: + logging.info( + "Popped request from the queue, beginning worker thread.", + extra={"request_id": id}, + ) + self.request.set_status(Status.PROCESSING) + self.update_status("processing", request_id=self.request.id) + self.request_store.update_request(self.request) + self.future = self.thread_pool.submit(self.process_request, (self.request)) + else: self.update_status("idle") - self.queue.ack(self.queue_msg) - # OK, process the request + # Future completed: do callback, ack message and reset state + elif self.future.done(): + try: + self.future.result(0) + except Exception as e: + self.on_request_fail(self.request, e) else: - logging.info( - "Popped request from the queue, beginning worker thread.", - extra={"request_id": id}, - ) - self.request.set_status(Status.PROCESSING) - self.update_status("processing", request_id=self.request.id) - self.request_store.update_request(self.request) - self.future = self.thread_pool.submit(self.process_request, (self.request)) - else: - self.update_status("idle") + self.on_request_complete(self.request) - # Future completed: do callback, ack message and reset state - elif self.future.done(): - try: - self.future.result(0) - except Exception as e: - self.on_request_fail(self.request, e) - else: - self.on_request_complete(self.request) - - self.queue.ack(self.queue_msg) + self.queue.ack(self.queue_msg) - self.update_status("idle") - self.request_store.update_request(self.request) + self.update_status("idle") + self.request_store.update_request(self.request) - self.future = None - self.queue_msg = None - self.request = None + self.future = None + self.queue_msg = None + self.request = None - # Future running: keep checking - else: - self.update_status("processing") + # Future running: keep checking + else: + self.update_status("processing") - self.update_metric() + # self.update_metric() + except Exception: + # We must force threads to shutdown in case of failure, otherwise the worker won't exit + self.thread_pool.shutdown(wait=False) + raise def process_request(self, request): """Entrypoint for the worker thread.""" @@ -245,9 +244,10 @@ def process_request(self, request): if datasource is not None: request.url = self.staging.create(id, datasource.result(request), datasource.mime_type()) - except Exception: - request.user_message += "Failed to finalize request" - logging.exception("Failed to finalize request", extra={"request_id": id}) + except Exception as e: + request.user_message += f"Failed to finalize request: [{str(type(e))}] {str(e)}" + logging.info(request.user_message, extra={"request_id": id}) + logging.exception("Failed to finalize request", extra={"request_id": id, "exception": str(e)}) raise # Guarantee destruction of the datasource @@ -257,6 +257,7 @@ def process_request(self, request): if datasource is None: request.user_message += "Failed to process request." + logging.info(request.user_message, extra={"request_id": id}) raise Exception("Failed to process request.") else: request.user_message += "Success" @@ -298,13 +299,15 @@ def on_request_fail(self, request, exception): """Called when the future thread raises an exception""" _, v, _ = sys.exc_info() + tb = traceback.format_exception(None, exception, exception.__traceback__) + logging.info(tb, extra={"request_id": request.id}) error_message = request.user_message + "\n" + str(v) request.set_status(Status.FAILED) request.user_message = error_message logging.exception("Request failed with exception.", extra={"request_id": request.id}) self.requests_failed += 1 - def on_process_terminated(self): + def on_process_terminated(self, signumm=None, frame=None): """Called when the worker is asked to exit whilst processing a request, and we want to reschedule the request""" if self.request is not None: diff --git a/pyproject.toml b/pyproject.toml index e34796e..85c3b07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,5 @@ [tool.black] -line-length = 120 \ No newline at end of file +line-length = 120 + +[tool.isort] +profile = "black" diff --git a/requirements.txt b/requirements.txt index e461a9d..9c578e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,32 +1,34 @@ -Jinja2==3.1.1 -Flask==2.0.3 -flask-script==2.0.5 -flask-mongoengine==0.8.2 -py-bcrypt==0.4 -jsonschema==2.5.1 -pika==1.1.0 -requests==2.22.0 attrdict==2.0.1 -pytest>=5.0.1 -pyyaml==5.1.1 -hiyapyco==0.4.14 -pykwalify==1.7.0 -falcon==2.0.0 -falcon-jsonify==1.2 -flask-restful==0.3.7 -flask-restplus==0.12.1 -flask-wtf==0.14.2 -werkzeug==2.0 -gunicorn==19.9.0 -ecmwf-api-client==1.5.4 -pymongo==3.10.1 -pymemcache==3.0.0 -redis==3.4.1 -markdown==3.2.1 -minio==5.0.7 -tiny-kubernetes==1.1.1 -deepmerge==0.1.0 -flask-swagger-ui==3.25.0 -ldap3==2.7 -docker==4.2.0 -python-keycloak==0.24.0 \ No newline at end of file +boto3==1.35.19 +covjsonkit==0.0.22 +deepmerge==2.0 +docker==7.1.0 +ecmwf-api-client==1.6.3 +Flask==3.0.3 +flask-mongoengine==1.0.0 +Flask-RESTful==0.3.10 +flask-restplus==0.13.0 +Flask-Script==2.0.6 +flask-swagger-ui==4.11.1 +Flask-WTF==1.2.1 +gunicorn==23.0.0 +HiYaPyCo==0.6.1 +Jinja2==3.1.4 +jsonschema==4.23.0 +ldap3==2.9.1 +Markdown==3.7 +minio==7.2.8 +pika==1.3.2 +polytope-mars==0.0.10 +polytope-python==1.0.7 +py-bcrypt==0.4 +pykwalify==1.8.0 +pymemcache==4.0.0 +pymongo==4.8.0 +pytest==8.3.3 +python-jose==3.3.0 +python-keycloak==4.4.0 +PyYAML==6.0.2 +redis==5.0.8 +requests==2.32.3 +Werkzeug==3.0.4 diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..8fef254 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,49 @@ + +apiVersion: skaffold/v4beta10 +kind: Config + +# This Skaffold configuration expects to find a skaffold.env file in the current directory. +# It should contain the following environment variables: +# - SKAFFOLD_DEFAULT_REPO: The default repository to use for images +# - rpm_repo: The URL of the RPM repository for MARS/GribJump/FDB images +# - mars_config_repo: The URL of the MARS configuration repository +# - mars_config_branch: The branch of the MARS configuration repository to use +# - ssh_pub_key: The public SSH key to use for cloning the MARS and MARS configuration repositories +# - ssh_prv_key: The private SSH key to use for cloning the MARS and MARS configuration repositories + + +build: + local: + useBuildkit: True + concurrency: 1 + + tagPolicy: + gitCommit: + ignoreChanges: true + + artifacts: + + # Polytope common + - image: "polytope-common" + docker: + target: polytope-common + buildArgs: + developer_mode: "{{ .developer_mode }}" + + # Worker with all clients (FDB, GribJump, MARS C, MARS CPP) + - image: "worker" + docker: + target: worker + buildArgs: + rpm_repo: "{{ .rpm_repo }}" + mars_config_repo: "{{ .mars_config_repo }}" + mars_config_branch: "{{ .mars_config_branch }}" + ssh_pub_key: "{{ .ssh_pub_key }}" + ssh_prv_key: "{{ .ssh_prv_key }}" + developer_mode: "{{ .developer_mode }}" + mars_client_cpp_version: 6.99.3.0 + mars_base_c: mars-base-c + mars_base_cpp: mars-base-cpp + fdb_base: blank-base #fdb-base + gribjump_base: gribjump-base + diff --git a/tests/unit/test_mongo_client_factory.py b/tests/unit/test_mongo_client_factory.py new file mode 100644 index 0000000..81ffe81 --- /dev/null +++ b/tests/unit/test_mongo_client_factory.py @@ -0,0 +1,44 @@ +import typing +from unittest import mock + +from polytope_server.common import mongo_client_factory + + +@mock.patch("polytope_server.common.mongo_client_factory.pymongo.MongoClient", autospec=True) +def test_create_without_credentials(mock_mongo: mock.Mock): + mongo_client_factory.create_client("mongodb://host:123") + + _verify(mock_mongo, "mongodb://host:123", None, None) + + +@mock.patch("polytope_server.common.mongo_client_factory.pymongo.MongoClient", autospec=True) +def test_create_without_password_credentials(mock_mongo: mock.Mock): + mongo_client_factory.create_client("mongodb+srv://host:123", username="admin") + + _verify(mock_mongo, "mongodb+srv://host:123", None, None) + + +@mock.patch("polytope_server.common.mongo_client_factory.pymongo.MongoClient", autospec=True) +def test_create_without_username_credentials(mock_mongo: mock.Mock): + mongo_client_factory.create_client("host:123", password="password") + + _verify(mock_mongo, "host:123", None, None) + + +@mock.patch("polytope_server.common.mongo_client_factory.pymongo.MongoClient", autospec=True) +def test_create_with_credentials(mock_mongo: mock.Mock): + mongo_client_factory.create_client("mongodb+srv://host", username="admin", password="est123123") + + _verify(mock_mongo, "mongodb+srv://host", "admin", "est123123") + + +def _verify( + mock_mongo: mock.Mock, endpoint: str, username: typing.Optional[str] = None, password: typing.Optional[str] = None +): + mock_mongo.assert_called_once() + args, kwargs = mock_mongo.call_args + assert args[0] == endpoint + if username: + assert kwargs["username"] == username + if password: + assert kwargs["password"] == password diff --git a/tests/unit/test_s3_staging.py b/tests/unit/test_s3_staging.py new file mode 100644 index 0000000..40f6cd1 --- /dev/null +++ b/tests/unit/test_s3_staging.py @@ -0,0 +1,50 @@ +from unittest import mock + +from polytope_server.common.staging.s3_staging import S3Staging + + +class DummyMinioClient: + def __init__(self) -> None: + self._region = None + + def make_bucket(self, bucket, region): + return "Dummy make bucket" + + def set_bucket_policy(self, bucket, policy): + return "Dummy set bucket policy" + + +class Test: + @mock.patch("polytope_server.common.staging.s3_staging.Minio", autospec=True) + def test_s3_staging_secure_false(self, mock_minio: mock.Mock): + mock_minio.return_value = DummyMinioClient() + s3Staging = S3Staging(config={"secure": False}) + + self.verify_secure_flag_and_internal_url(mock_minio, s3Staging, False) + + @mock.patch("polytope_server.common.staging.s3_staging.Minio", autospec=True) + def test_s3_staging_secure_any_value_false(self, mock_minio: mock.Mock): + mock_minio.return_value = DummyMinioClient() + s3Staging = S3Staging(config={"secure": "sdafsdfs"}) + + self.verify_secure_flag_and_internal_url(mock_minio, s3Staging, False) + + @mock.patch("polytope_server.common.staging.s3_staging.Minio", autospec=True) + def test_s3_staging_secure_default(self, mock_minio: mock.Mock): + mock_minio.return_value = DummyMinioClient() + s3Staging = S3Staging(config={}) + + self.verify_secure_flag_and_internal_url(mock_minio, s3Staging, False) + + @mock.patch("polytope_server.common.staging.s3_staging.Minio", autospec=True) + def test_s3_staging_secure_true(self, mock_minio: mock.Mock): + mock_minio.return_value = DummyMinioClient() + s3Staging = S3Staging(config={"secure": True}) + + self.verify_secure_flag_and_internal_url(mock_minio, s3Staging, True) + + def verify_secure_flag_and_internal_url(self, mock_minio: mock.Mock, s3Staging: S3Staging, secure: bool): + mock_minio.assert_called_once() + _, kwargs = mock_minio.call_args + assert kwargs["secure"] == secure + assert s3Staging.get_internal_url("test").startswith("https" if secure else "http") diff --git a/tox.ini b/tox.ini index 754e5f9..7fc5f9c 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,7 @@ exclude = .* extend-ignore = E203,W503 per-file-ignores = */__init__.py: F401,F403 + polytope_server/common/datasource/polytope.py: E402,F401 [isort] profile=black skip_glob=.* \ No newline at end of file