From dc131368a28e4c2cceeaa9a03aa0483b6b9858e5 Mon Sep 17 00:00:00 2001 From: "Antonovic P. Milan" Date: Tue, 13 Jul 2021 17:34:03 +0200 Subject: [PATCH] CSV converted improvements and small improvements --- README.md | 46 ++++++++++++ scripts/Dockerfile | 4 ++ scripts/README.md | 25 +++++++ scripts/converter/csv.py | 149 ++++++++++++++++++++++++++++++++------- scripts/raw2csv.py | 2 + scripts/requirements.txt | 1 + 6 files changed, 200 insertions(+), 27 deletions(-) create mode 100644 scripts/Dockerfile create mode 100644 scripts/README.md diff --git a/README.md b/README.md index 8ee7bcb3..90842ee9 100755 --- a/README.md +++ b/README.md @@ -8,3 +8,49 @@ The project provides also a Graphical user Interface that allows for easing the For further information, please refer to the istSOS website: istSOS is released under the GPL License, and runs on all major platforms (Windows, Linux, Mac OS X), even though tests were conducted under a Linux environment. + + +## Start istSOS with docker-compose + +The faste way to use istSOS is with docker. + +Create a docker-compose.yml file: + +```bash +version: '3.7' + +services: + istsos-db: + image: postgis/postgis:12-2.5-alpine + restart: always + container_name: istsos-db + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: istsos + TZ: Europe/Zurich + volumes: + - istsos-db-data:/var/lib/postgresql/data + + istsos: + image: istsos/istsos:2.4.1 + container_name: istsos + restart: always + ports: + - 80:80 + +volumes: + istsos-db-data: + name: istsos-db-data + +``` + +And start the containers by running the following command: + + +```bash +docker-compose up -d +``` + +Open your browser and go to http://localhost/istsos/admin you should be able to see the web admin page. +Before creating a new istsos service instance go to the "Database" page and set "istsos-db" as Host. \ No newline at end of file diff --git a/scripts/Dockerfile b/scripts/Dockerfile new file mode 100644 index 00000000..dde403fa --- /dev/null +++ b/scripts/Dockerfile @@ -0,0 +1,4 @@ + +FROM python:2.7.18-slim-buster +COPY ./ /app/scripts +RUN pip install -r /app/scripts/requirements.txt diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..1e66b810 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,25 @@ +# Create the virtualenv with docker + +Build the images + +```bash +docker build --no-cache -t docker push docker.pkg.github.com/istsos/istsos2/istsos-scripts:2.4.1 . +``` + +Publish + +```bash +docker push docker.pkg.github.com/istsos/istsos2/istsos-scripts:2.4.1 +``` + +Run the script +```bash +docker run --rm -it \ + --network="host" \ + -v $PWD/acquisition:/app/acquisition \ + -v /home/milan/workspace/istsos/istsos2/scripts:/app/scripts \ + -v /mnt/acq/:/mnt/acq/ \ + -w /app \ + istsos/istsos-scripts:2.4.1 \ + python ./acquisition/acquisition.py +``` diff --git a/scripts/converter/csv.py b/scripts/converter/csv.py index 127928e5..7b8f572e 100755 --- a/scripts/converter/csv.py +++ b/scripts/converter/csv.py @@ -98,6 +98,61 @@ ) csv.execute() + +# File example: BR7_GW_____20210325000036.MIS +# ===================================== +BR7_GW____0001YYYYMMDD +20201024;181000;9.997 +20201024;182000;9.996 +20201024;183000;9.996 +BR7_GW____0002YYYYMMDD +20201024;181000;8.68 +20201024;182000;8.70 +20201024;183000;8.68 +# ===================================== + +# Block interval enable / disable +importer = csv.CsvImporter('T_TRE', { + "enabled_from": { + "include": "0001" + }, + "disabled_from": { + "include": "", + "exclude": "0001" + }, + skiplines: [ + 'NO DATA AVAILABLE' + ], + "headrows": 0, + "separator": ";", + "filenamedate": { + "format": '%Y%m%d%H%M%S', + "remove": ['BR7_GW_____','.MIS'] + }, + "datetime": { + "tz": '+01:00', + "time": { + "column": 1, + "format": '%H%M%S' + }, + "date": { + "column": 2, + "format": '%Y%m%d' + } + }, + "observations": [{ + "observedProperty": "urn:ogc:def:parameter:x-istsos:1.0:meteo:air:temperature", + "column": 2 + }] + }, + 'http://localhost/istsos', 'pippo', + "istsos/test/scripts/data/in/csv", 'BR7_GW_____20210325000036.MIS', + "istsos/test/scripts/data/out", + True +) +csv.execute() + + """ from scripts import raw2csv @@ -116,7 +171,19 @@ def __init__(self, procedureName, config, url, service, inputDir, csvlength=5000, filenamecheck=None, archivefolder = None, extra={}): self.config = config - + + self.max_cols = 0 + + if 'observations' in config: + for obs in config['observations']: + if 'column' in obs: + self.max_cols = max(self.max_cols, obs['column']) + + if 'enabled_from' in config: + self.enabled = False + else: + self.enabled = None + raw2csv.Converter.__init__(self, procedureName, url, service, inputDir, fileNamePattern, outputDir, qualityIndex, exceptionBehaviour, user, password, debug, csvlength, @@ -187,8 +254,6 @@ def setEndPositionFromFilename(self, fileName): self.setEndPosition(dt) """ - - def parse(self, fileObj, fileName): # print "Filename: %s" % fileName @@ -197,37 +262,67 @@ def parse(self, fileObj, fileName): cnt = cnt+1 # Skipping header rows if present in configuration if "headrows" in self.config and cnt <= self.config['headrows']: + # print("skipping headrows (%s/%s)" % (cnt, self.config['headrows'])) continue elif "stopat" in self.config: if isinstance(self.config["stopat"], str) and ( line.find(self.config["stopat"])>=0): + # print("Breaking at line %s: %s" % (cnt, line)) break - elif isinstance(self.config["stopat"], int): - pass - - else: - pass - - # Line splitting - columns = line.split(self.config['separator']) - try: - date = self.parseDate(columns) - values = {} - for obs in self.config["observations"]: - if obs["column"] is None: - values[obs["observedProperty"]] = -999.9 - else: - values[obs["observedProperty"]] = columns[obs["column"]] - self.addObservation( - raw2csv.Observation(date, values) - ) - self.setEndPosition(date) - except Exception as e: - print("%s [%s]:%s" % (fileName,cnt,line)) - print(traceback.print_exc()) - raise e + if "skiplines" in self.config: + skipline = False + for sl in self.config['skiplines']: + if sl in line: + skipline = True + break + + if skipline is True: + # print("Skipping at line %s, found '%s': %s" % (cnt, sl, line)) + continue + + if self.enabled is not None and self.enabled is not True: + + if self.config['enabled_from']['include'] in line: + # print("Enabled at line %s, found '%s': %s" % (cnt, self.config['enabled_from']['include'], line)) + self.enabled = True + continue + + elif self.enabled is not None and self.enabled is True: + + if ( + self.config['disabled_from']['include'] in line + and self.config['disabled_from']['exclude'] not in line + ): + self.enabled = False + continue + + if self.enabled is None or self.enabled is True: + + # Line splitting + columns = line.split(self.config['separator']) + + if self.max_cols > 0 and len(columns) < (self.max_cols+1): + continue + + try: + date = self.parseDate(columns) + values = {} + for obs in self.config["observations"]: + if obs["column"] is None: + values[obs["observedProperty"]] = -999.9 + else: + values[obs["observedProperty"]] = columns[obs["column"]] + self.addObservation( + raw2csv.Observation(date, values) + ) + self.setEndPosition(date) + + except Exception as e: + print("%s [%s]:%s" % (fileName,cnt,line)) + print(traceback.print_exc()) + raise e self.setEndPositionFromFilename(fileName) diff --git a/scripts/raw2csv.py b/scripts/raw2csv.py index c68eae92..fcdf729e 100755 --- a/scripts/raw2csv.py +++ b/scripts/raw2csv.py @@ -32,6 +32,7 @@ import os from os import path import glob +import fnmatch from datetime import datetime from datetime import timedelta import decimal @@ -591,6 +592,7 @@ def prepareFiles(self): self.addException(msg) raise FileReaderError(msg) + print(" > Searching: %s" % os.path.join(self.folderIn, "%s" % (self.pattern))) files = list(filter( path.isfile, glob.glob(os.path.join(self.folderIn, "%s" % (self.pattern))))) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 919ef2e7..bbbbab81 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -6,3 +6,4 @@ pytz==2019.3 requests==2.23.0 six==1.14.0 urllib3[secure]==1.25.7 +html==1.16 \ No newline at end of file