diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index c73e032..7e0958a 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -18,6 +18,7 @@ jobs: run: | python -m pip install --upgrade pip pip install pylint + pip install -r app/requirements.txt - name: Analysing the code with pylint run: | pylint $(git ls-files '*.py') diff --git a/src/import/article.py b/src/import/article.py index edf911f..3214f40 100644 --- a/src/import/article.py +++ b/src/import/article.py @@ -1,19 +1,24 @@ +""" +article.py +Contains Article class used for importing Usenet News articles. +""" + +import email import json from dateutil import parser -import email -class Article(object): +class Article: """ Represents a single article. """ - def __init__(self, id, folder, path, from_raw, newsgroups, subject, message_id, date, x_gateway, lines, xref, + def __init__(self, articleId, folder, path, from_raw, newsgroups, subject, message_id, date, x_gateway, lines, xref, body, references): self.from_name = None self.from_email = None self.date = None - self.id = id + self.articleId = articleId self.location = folder self.path = path self.newsgroups = newsgroups.split(',') @@ -73,7 +78,7 @@ def to_dict(self): :return: """ return { - 'id': self.id, + 'id': self.articleId, 'path': self.path, 'folder': self.location, 'from_name': self.from_name, @@ -120,9 +125,9 @@ def from_file(path): pathstr = str(path) path_parts = pathstr.split('/') - id = '-'.join(path_parts[-3:]) + article_id = '-'.join(path_parts[-3:]) return Article( - id, + article_id, '/'.join(path_parts[-3:-1]), msg['Path'], msg['From'], diff --git a/src/import/import.py b/src/import/import.py deleted file mode 100644 index 63f77b6..0000000 --- a/src/import/import.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python3 - diff --git a/src/import/test_import.py b/src/import/test_import.py index 84a5a03..f193027 100755 --- a/src/import/test_import.py +++ b/src/import/test_import.py @@ -1,7 +1,13 @@ #!/usr/bin/env python3 +""" +This is a test import script which imports data from zipped +Usenet news files. +""" + from article import Article import os +import sys from pathlib import Path from elasticsearch import Elasticsearch, helpers import tempfile @@ -20,12 +26,12 @@ def create_mapping(name): if client.indices.exists(name): client.indices.delete(index=name) - with open("../service/fields.yaml") as stream: + with open("../service/fields.yaml", encoding="utf-8") as stream: try: data = yaml.safe_load(stream) except yaml.YAMLError as e: print(e) - exit(1) + sys.exit(1) properties = {} diff --git a/src/service/app.py b/src/service/app.py index 8858a76..0097725 100644 --- a/src/service/app.py +++ b/src/service/app.py @@ -1,7 +1,7 @@ -from flask import Flask, request, jsonify -from flask_cors import CORS import os from elastic_index import Index +from flask import Flask, request, jsonify +from flask_cors import CORS import yaml app = Flask(__name__, static_folder="browser", static_url_path="") @@ -28,16 +28,28 @@ def after_request(response): @app.route("/", methods=["GET", "POST"]) @app.route("/search") def catch_all(): + """ + Return the front-end, pages are handled by React + :return: + """ return app.send_static_file("index.html") -@app.route("/api/detail/") +@app.route("/detail/") def detail(id): + """ + Return the front-end, pages are handled by React + :return: + """ return app.send_static_file("index.html") @app.route("/api/facets", methods=["GET"]) def get_facets(): + """ + Get all used facets, and their configuration. + :return: + """ try: data = index.get_facets() except yaml.YAMLError as e: @@ -49,6 +61,10 @@ def get_facets(): @app.route("/api/facet", methods=["POST", "GET"]) def get_facet(): + """ + Get facet information. + :return: + """ struc = request.get_json() ret_struc = index.get_facet(struc["name"], struc["amount"], struc["filter"], struc["searchvalues"]) return jsonify(ret_struc) @@ -56,6 +72,10 @@ def get_facet(): @app.route("/api/browse", methods=["POST", "GET"]) def browse(): + """ + Search for articles using elasticsearch. + :return: + """ struc = request.get_json() ret_struc = index.browse(struc["page"], struc["page_length"], struc["searchvalues"]) return jsonify(ret_struc) @@ -63,5 +83,9 @@ def browse(): @app.route("/api/article", methods=["GET"]) def get_article(): - id = request.args.get("rec") - return jsonify(index.by_id(id)) + """ + Get details of a single article. + :return: + """ + article_id = request.args.get("rec") + return jsonify(index.by_id(article_id))