From 9aa425090c523d2d2bdd556e039479f72a59c999 Mon Sep 17 00:00:00 2001 From: Samuel Cantero Date: Thu, 22 Jun 2017 08:31:48 -0400 Subject: [PATCH 01/57] Refactor app * Use blueprints * Add default_settings config file * Add gunicorn * Add honcho --- .gitignore | 102 +------------------------------ LICENSE | 21 ------- Procfile | 2 + README.md | 37 ----------- activity-build-docker/Dockerfile | 16 +++++ app.py | 35 ----------- aslo/__init__.py | 36 +++++++++++ aslo/api/__init__.py | 5 ++ aslo/api/build.py | 93 ++++++++++++++++++++++++++++ aslo/api/errors.py | 12 ++++ aslo/api/exceptions.py | 18 ++++++ aslo/api/tasks.py | 24 ++++++++ aslo/api/utils.py | 7 +++ aslo/api/views.py | 33 ++++++++++ aslo/celery_app.py | 28 +++++++++ aslo/default_settings.py | 37 +++++++++++ aslo/web/__init__.py | 5 ++ aslo/web/templates/index.html | 36 +++++++++++ aslo/web/views.py | 7 +++ gunicorn_config.py | 11 ++++ requirements.txt | 23 +++---- start.sh | 2 + utils.py | 13 ---- worker.py | 3 + wsgi.py | 7 +++ 25 files changed, 391 insertions(+), 222 deletions(-) delete mode 100644 LICENSE create mode 100644 Procfile delete mode 100644 README.md create mode 100644 activity-build-docker/Dockerfile delete mode 100644 app.py create mode 100644 aslo/__init__.py create mode 100644 aslo/api/__init__.py create mode 100644 aslo/api/build.py create mode 100644 aslo/api/errors.py create mode 100644 aslo/api/exceptions.py create mode 100644 aslo/api/tasks.py create mode 100644 aslo/api/utils.py create mode 100644 aslo/api/views.py create mode 100644 aslo/celery_app.py create mode 100644 aslo/default_settings.py create mode 100644 aslo/web/__init__.py create mode 100644 aslo/web/templates/index.html create mode 100644 aslo/web/views.py create mode 100644 gunicorn_config.py create mode 100644 start.sh delete mode 100644 utils.py create mode 100644 worker.py create mode 100644 wsgi.py diff --git a/.gitignore b/.gitignore index 7bbc71c..c4672c2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,101 +1,3 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv +*.pyc +env .env - -# virtualenv -.venv -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 5e19011..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 Jatin Dhankhar - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/Procfile b/Procfile new file mode 100644 index 0000000..afab67d --- /dev/null +++ b/Procfile @@ -0,0 +1,2 @@ +site: gunicorn -c gunicorn_config.py wsgi +worker: celery -A worker worker diff --git a/README.md b/README.md deleted file mode 100644 index e40b295..0000000 --- a/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# aslo-v3 -Upcoming Software center for SugarLabs (ASLO V3) - - -## Setup Instructions - -### Install virtualenv - -`pip install virtualenv` - -### Create a vritualenv -`virtualenv ~/envs/flask-dev` - -### Activate virtualenv -`source ~/envs/flask-dev/bin/activate` - -### Install Dependencies -`pip install -r requirements.txt` - -### Install Redis - -``` bash - -cd /tmp -wget http://download.redis.io/releases/redis-stable.tar.gz -tar xzf redis-stable.tar.gz -make -make test -sudo make install -``` - For a more comprehensive guide follow this [Tutorial from DigitalOcean](https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-redis-on-ubuntu-16-04) - -Make sure redis is running by logging in `redis-cli` - -### Run the Flask Server - -`python app.py` will start Flask server listening on all interfaces `0.0.0.0` and port `5000` diff --git a/activity-build-docker/Dockerfile b/activity-build-docker/Dockerfile new file mode 100644 index 0000000..4b2a560 --- /dev/null +++ b/activity-build-docker/Dockerfile @@ -0,0 +1,16 @@ +FROM fedora:22 + +RUN dnf install -y git gettext + +RUN git clone --depth=1 https://github.com/sugarlabs/sugar-toolkit-gtk3; \ + mv sugar-toolkit-gtk3/src/sugar3 /usr/lib/python2.7/site-packages/sugar3; \ + rm -rf sugar-toolkit-gtk3 + +RUN rm -rf /usr/lib/python2.7/site-packages/sugar; \ + ln -s /usr/lib/python2.7/site-packages/sugar3 /usr/lib/python2.7/site-packages/sugar + +VOLUME /activity +WORKDIR /activity + +CMD python /activity/setup.py dist_xo --no-fail; \ + chmod 777 -R /activity/ diff --git a/app.py b/app.py deleted file mode 100644 index 48fc0a2..0000000 --- a/app.py +++ /dev/null @@ -1,35 +0,0 @@ -from flask import Flask,request,abort -from celery import Celery -import os -from IPython import embed -import utils - -app = Flask(__name__) -app.config['CELERY_BROKER_URL'] = 'redis://localhost:6379/0' -app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost:6379/0' - -celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL']) -celery.conf.update(app.config) -# Set false for production systems -app.debug = True -GITHUB_HOOK_SECRET = os.environ.get('GITHUB_HOOK_SECRET') - -@app.route('/') -def main(): - return "Welcome" - -@app.route('/webhook',methods=['POST']) -def handle_payload(): - content = request.get_json(silent=True) - #embed() - header_signature = request.headers['X-Hub-Signature'] - embed() - if header_signature is None: - abort(403) - if not utils.verify_signature(header_signature,request.data,GITHUB_HOOK_SECRET): - abort(403) - - return "Authenticated request :D" - -if __name__ == "__main__": - app.run(host='0.0.0.0') diff --git a/aslo/__init__.py b/aslo/__init__.py new file mode 100644 index 0000000..760ac13 --- /dev/null +++ b/aslo/__init__.py @@ -0,0 +1,36 @@ +from flask import Flask +from flask_bootstrap import Bootstrap +import logging + +logger = logging.getLogger(__name__) + + +def init_app(): + app = Flask(__name__) + app.config.from_object('aslo.default_settings') + + # init bootstrap + Bootstrap(app) + + # init celery + from .celery_app import init_celery + init_celery(app) + + # blueprints + from .web import web + app.register_blueprint(web) + + from .api import api + app.register_blueprint(api, url_prefix='/api') + + # logging + logger.setLevel(logging.INFO) + fmt = logging.Formatter('[%(asctime)s] %(levelname).3s %(message)s') + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(fmt) + logger.addHandler(stream_handler) + + if app.config['DEBUG']: + logger.setLevel(logging.DEBUG) + + return app diff --git a/aslo/api/__init__.py b/aslo/api/__init__.py new file mode 100644 index 0000000..a7b4fbb --- /dev/null +++ b/aslo/api/__init__.py @@ -0,0 +1,5 @@ +from flask import Blueprint + +api = Blueprint('api', __name__) + +from . import views, errors # noqa diff --git a/aslo/api/build.py b/aslo/api/build.py new file mode 100644 index 0000000..dffd150 --- /dev/null +++ b/aslo/api/build.py @@ -0,0 +1,93 @@ +import os +import shutil +import configparser +from flask import current_app as app +from aslo.celery_app import logger +from subprocess import call +from .exceptions import BuildProcessError + + +def get_repo_location(name): + return os.path.join(app.config['BUILD_CLONE_REPO'], name) + + +def clone_repo(url, name, tag): + target_dir = app.config['BUILD_CLONE_REPO'] + if not os.path.isdir(target_dir): + raise BuildProcessError('Directory %s does not exist' % target_dir) + + if os.path.isdir(get_repo_location(name)): + logger.info('Removing existing cloned repo for %s', name) + shutil.rmtree(get_repo_location(name)) + + cmd = ['git', '-c', 'advice.detachedHead=false', '-C', target_dir, + 'clone', '-b', tag, '--depth', '1', url] + + logger.info('Cloning repo %s', url) + if call(cmd) != 0: + raise BuildProcessError('[%s] command has failed' % ' '.join(cmd)) + + +def get_activity_metadata(name): + def metadata_file_exists(): + repo_dir = get_repo_location(name) + activity_file = os.path.join(repo_dir, "activity/activity.info") + if not os.path.isfile(activity_file): + raise BuildProcessError( + 'Activity file %s does not exist', activity_file + ) + + return activity_file + + def parse_metadata_file(): + parser = configparser.ConfigParser() + if len(parser.read(activity_file)) == 0: + raise BuildProcessError('Error parsing metadata file') + + try: + attributes = dict(parser.items('Activity')) + except configparser.NoSectionError as e: + raise BuildProcessError( + 'Error parsing metadata file. Exception message: %s', e + ) + + return attributes + + activity_file = metadata_file_exists() + return parse_metadata_file() + + +def invoke_build(name): + def store_bundle(): + dist_dir = os.path.join(get_repo_location(name), 'dist') + if os.path.isdir(dist_dir) and len(os.listdir(dist_dir)) == 1: + bundle_name = os.path.join(dist_dir, os.listdir(dist_dir)[0]) + else: + raise BuildProcessError('Bundle file was not generated correctly') + + try: + shutil.copy2(bundle_name, app.config['BUILD_BUNDLE_DIR']) + stored_bundle = os.path.join( + app.config['BUILD_BUNDLE_DIR'], + os.path.basename(bundle_name) + ) + os.chmod(stored_bundle, 0o644) + except IOError as e: + raise BuildProcessError( + 'Bundle copying has failed: %s', e + ) + + logger.info('Bundle succesfully stored at %s', stored_bundle) + + def clean(): + shutil.rmtree(get_repo_location(name)) + + volume = get_repo_location(name) + ':/activity' + docker_image = app.config['BUILD_DOCKER_IMAGE'] + docker_cmd = ['docker', 'run', '--rm', '-v', volume, docker_image] + logger.info('Running docker command: "%s"', ' '.join(docker_cmd)) + if call(docker_cmd) != 0: + raise BuildProcessError('Docker building process has failed') + + store_bundle() + clean() diff --git a/aslo/api/errors.py b/aslo/api/errors.py new file mode 100644 index 0000000..81951be --- /dev/null +++ b/aslo/api/errors.py @@ -0,0 +1,12 @@ +import json +from flask import make_response +from . import api +from .exceptions import ApiHttpError + + +@api.errorhandler(ApiHttpError) +def HandleHttpApiError(e): + error_msg = json.dumps(e.to_dict()) + response = make_response(error_msg, e.status_code) + response.headers['Content-Type'] = 'application/json' + return response diff --git a/aslo/api/exceptions.py b/aslo/api/exceptions.py new file mode 100644 index 0000000..b2f6fab --- /dev/null +++ b/aslo/api/exceptions.py @@ -0,0 +1,18 @@ + + +class BuildProcessError(Exception): + pass + + +class ApiHttpError(Exception): + + def __init__(self, message, status_code=None): + Exception.__init__(self) + self.message = message + self.status_code = status_code if status_code else 400 + + def to_dict(self): + return { + 'message': self.message, + 'status_code': self.status_code + } diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py new file mode 100644 index 0000000..569d0c5 --- /dev/null +++ b/aslo/api/tasks.py @@ -0,0 +1,24 @@ +from . import build +from aslo.celery_app import celery, logger +from .exceptions import BuildProcessError + + +@celery.task(bind=True) +def build_process(self, gh_json): + try: + url = gh_json['repository']['clone_url'] + name = gh_json['repository']['name'] + tag = gh_json['release']['tag_name'] + + # TODO: work on cases (bundle attached or not) + # and insert data into DB + + build.clone_repo(url, name, tag) + activity = build.get_activity_metadata(name) + build.invoke_build(name) + + except BuildProcessError as e: + logger.exception("Error in activity building process") + return False + + logger.info('Activity building process finished succesfully!') diff --git a/aslo/api/utils.py b/aslo/api/utils.py new file mode 100644 index 0000000..7cf9da2 --- /dev/null +++ b/aslo/api/utils.py @@ -0,0 +1,7 @@ +import hmac +import hashlib + + +def verify_signature(gh_signature, body, secret): + sha1 = hmac.new(secret.encode(), body, hashlib.sha1).hexdigest() + return hmac.compare_digest('sha1=' + sha1, gh_signature) diff --git a/aslo/api/views.py b/aslo/api/views.py new file mode 100644 index 0000000..2ff250e --- /dev/null +++ b/aslo/api/views.py @@ -0,0 +1,33 @@ +from flask import request, current_app as app +from . import api +from .exceptions import ApiHttpError +from .tasks import build_process +from .utils import verify_signature + + +@api.route('/hook', methods=['POST']) +def hook(): + if request.is_json: + body_json = request.get_json() + else: + raise ApiHttpError( + 'Invalid Content-type. application/json expected.', 400 + ) + + if 'X-Hub-Signature' not in request.headers: + raise ApiHttpError( + 'X-Hub-Signature header required.', 400 + ) + + valid_signature = verify_signature( + request.headers['X-Hub-Signature'], + request.data, + app.config['GITHUB_HOOK_SECRET'] + ) + + if not valid_signature: + raise ApiHttpError('Invalid Signature', 400) + + build_process.apply_async(args=[body_json]) + + return "{'status_code': 200, 'message': 'OK'}", 200 diff --git a/aslo/celery_app.py b/aslo/celery_app.py new file mode 100644 index 0000000..0cb4691 --- /dev/null +++ b/aslo/celery_app.py @@ -0,0 +1,28 @@ +import logging +from celery import Celery +from celery.utils.log import get_task_logger + +celery = Celery(__name__) +TaskBase = celery.Task +logger = get_task_logger(__name__) + + +def init_celery(app): + class ContextTask(TaskBase): + abstract = True + + def __call__(self, *args, **kwargs): + with app.app_context(): + return TaskBase.__call__(self, *args, **kwargs) + +# def on_failure(self, exc, task_id, args, kwargs, einfo): +# print('{0!r} failed: {1!r}'.format(task_id, exc)) + + celery.Task = ContextTask + celery.config_from_object(app.config, namespace='CELERY') + app.celery = celery + + # logging + logger.setLevel(logging.INFO) + if app.config['DEBUG']: + logger.setLevel(logging.DEBUG) diff --git a/aslo/default_settings.py b/aslo/default_settings.py new file mode 100644 index 0000000..01ec739 --- /dev/null +++ b/aslo/default_settings.py @@ -0,0 +1,37 @@ +import os + + +def env(variable, fallback_value=None): + env_value = os.environ.get(variable) + if env_value is None: + return fallback_value + # needed for honcho + elif env_value == "__EMPTY__": + return '' + else: + return env_value + + +# FLASK +DEBUG = env('DEBUG', False) +SECRET_KEY = env('SECRET_KEY', '') + +# GITHUB WEBHOOK +GITHUB_HOOK_SECRET = env('GITHUB_HOOK_SECRET', '') + +# CELERY +REDIS_URI = env('REDIS_URL', 'redis://localhost:6379/1') +CELERY_BROKER_URL = env('CELERY_BROKER_URL', REDIS_URI) +CELERY_TASK_IGNORE_RESULT = True + +# MONGO +MONGO_DBNAME = env('MONGO_DBNAME', 'aslo') +MONGO_URI = env('MONGO_URI', 'mongodb://localhost/%s' % MONGO_DBNAME) + +# BUILD +# Docker image name for activity building containers. +BUILD_DOCKER_IMAGE = env('BUILD_DOCKER_IMAGE', 'sugar-activity-build') +# Base path for cloned activities +BUILD_CLONE_REPO = env('BUILD_CLONE_REPO', '/var/tmp/activities/') +# Path where bundles are going to be stored +BUILD_BUNDLE_DIR = env('BUILD_BUNDLE_DIR', '/srv/activities/') diff --git a/aslo/web/__init__.py b/aslo/web/__init__.py new file mode 100644 index 0000000..20d9574 --- /dev/null +++ b/aslo/web/__init__.py @@ -0,0 +1,5 @@ +from flask import Blueprint + +web = Blueprint('web', __name__, template_folder='templates') + +from . import views # noqa diff --git a/aslo/web/templates/index.html b/aslo/web/templates/index.html new file mode 100644 index 0000000..a3ce364 --- /dev/null +++ b/aslo/web/templates/index.html @@ -0,0 +1,36 @@ +{% extends "bootstrap/base.html" %} +{% block title %}Sugar Activities | SugarLabs{% endblock %} + +{% block navbar %} + +{% endblock %} + +{% block content %} +{% endblock %} diff --git a/aslo/web/views.py b/aslo/web/views.py new file mode 100644 index 0000000..886e873 --- /dev/null +++ b/aslo/web/views.py @@ -0,0 +1,7 @@ +from flask import render_template +from . import web + + +@web.route('/') +def index(): + return render_template('index.html') diff --git a/gunicorn_config.py b/gunicorn_config.py new file mode 100644 index 0000000..44cad3a --- /dev/null +++ b/gunicorn_config.py @@ -0,0 +1,11 @@ +import os +import multiprocessing + +bind = '0.0.0.0:%s' % os.environ.get('PORT', '5000') +workers = int( + os.environ.get('GUNICORN_WORKERS', multiprocessing.cpu_count()) +) +timeout = int(os.environ.get('G_TIMEOUT', 30)) + +accesslog = '-' +access_log_format = '%(m)s %(U)s status=%(s)s time=%(T)ss size=%(B)sb' diff --git a/requirements.txt b/requirements.txt index 53641e0..21e36d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,7 @@ -amqp==2.1.4 -appdirs==1.4.3 -billiard==3.5.0.2 -celery==4.0.2 -click==6.7 -Flask==0.12.2 -itsdangerous==0.24 -Jinja2==2.9.6 -kombu==4.0.2 -MarkupSafe==1.0 -packaging==16.8 -pyparsing==2.2.0 -pytz==2017.2 -six==1.10.0 -vine==1.1.3 -Werkzeug==0.12.2 +honcho +gunicorn +flask +flask_bootstrap +celery +redis +pymongo diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..5bdd6d4 --- /dev/null +++ b/start.sh @@ -0,0 +1,2 @@ +. env/bin/activate +exec honcho start diff --git a/utils.py b/utils.py deleted file mode 100644 index 3c27dbe..0000000 --- a/utils.py +++ /dev/null @@ -1,13 +0,0 @@ -import hmac -from hashlib import sha1 - -# Thanks to https://github.com/carlos-jenkins/python-github-webhooks/blob/master/webhooks.py -# https://developer.github.com/webhooks/securing/ -def verify_signature(header_signature,raw_data,secret): - sha_name, signature = header_signature.split('=') - if sha_name != 'sha1': - return false - # HMAC requires the key to be bytes, pass raw request data - mac = hmac.new(secret,raw_data,sha1) - # Use compare_digest to avoid timing attacks - return hmac.compare_digest(str(mac.hexdigest()), str(signature)) diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..9adbfd9 --- /dev/null +++ b/worker.py @@ -0,0 +1,3 @@ +from aslo import init_app + +celery = init_app().celery diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 0000000..3fb43fa --- /dev/null +++ b/wsgi.py @@ -0,0 +1,7 @@ +from aslo import init_app + +application = init_app() + +if __name__ == "__main__": + host = '0.0.0.0' + application.run(host=host) From 7de559a87fca7fff19263b72b7b31580b1ba5c1c Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Sat, 24 Jun 2017 22:36:37 +0530 Subject: [PATCH 02/57] Refactor code from devel branch --- .gitignore | 3 ++ aslo/__init__.py | 2 +- aslo/api/build.py | 79 ++++++++++++++++++++++++++++++++++++++++ aslo/api/tasks.py | 56 +++++++++++++++++++++++++--- aslo/default_settings.py | 2 + create_directories.sh | 22 +++++++++++ requirements.txt | 1 + start.sh | 2 +- 8 files changed, 160 insertions(+), 7 deletions(-) create mode 100755 create_directories.sh mode change 100644 => 100755 start.sh diff --git a/.gitignore b/.gitignore index c4672c2..3dfd16c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ *.pyc env .env +dist +bundles +repos diff --git a/aslo/__init__.py b/aslo/__init__.py index 760ac13..9b06388 100644 --- a/aslo/__init__.py +++ b/aslo/__init__.py @@ -22,7 +22,7 @@ def init_app(): from .api import api app.register_blueprint(api, url_prefix='/api') - + # logging logger.setLevel(logging.INFO) fmt = logging.Formatter('[%(asctime)s] %(levelname).3s %(message)s') diff --git a/aslo/api/build.py b/aslo/api/build.py index dffd150..dac5735 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -5,12 +5,57 @@ from aslo.celery_app import logger from subprocess import call from .exceptions import BuildProcessError +import requests +import zipfile def get_repo_location(name): return os.path.join(app.config['BUILD_CLONE_REPO'], name) +def get_bundle_path(bundle_name): + return os.path.join(app.config['BUILD_BUNDLE_DIR'], bundle_name) + + +def check_and_download_assets(assets): + + def download_asset(download_url, name): + response = requests.get(download_url, stream=True) + # Save with every block of 1024 bytes + logger.info("Downloading File .. " + name) + with open(app.config['TEMP_BUNDLE_DIR'] + "/" + name, "wb") as handle: + for block in response.iter_content(chunk_size=1024): + handle.write(block) + return + + def check_info_file(name): + logger.info("Checking For Activity.info") + xo_file = zipfile.ZipFile(app.config['TEMP_BUNDLE_DIR'] + "/" + name) + return any("activity.info" in filename for filename in xo_file.namelist()) + + def asset_name_check(asset_name): + print("Checking for presence of .xo in name of " + asset_name) + return ".xo" in asset_name + + def verify_bundle(bundle_name): + bundle_path = get_bundle_path(bundle_name) + return os.path.exists(bundle_path) and os.path.isfile(bundle_path) + + def asset_manifest_check(download_url, bundle_name): + download_asset(download_url, bundle_name) + if check_info_file(bundle_name): + # Check if that bundle already exists then we don't continue + # Return false if that particular bundle already exists + if verify_bundle(bundle_name): + os.remove(app.config['TEMP_BUNDLE_DIR'] + "/" + bundle_name) + raise BuildProcessError('File %s already exits' % bundle_name) + else: + shutil.move(app.config['TEMP_BUNDLE_DIR'] + "/" + + bundle_name, app.config['BUILD_BUNDLE_DIR']) + return bundle_name + return False + + def clone_repo(url, name, tag): target_dir = app.config['BUILD_CLONE_REPO'] if not os.path.isdir(target_dir): @@ -91,3 +136,37 @@ def clean(): store_bundle() clean() + + +def invoke_bundle_build(activity_file): + def parse_metadata_file(): + parser = configparser.ConfigParser() + if len(parser.read_string(activity_file)) == 0: + raise BuildProcessError('Error parsing metadata file') + + try: + attributes = dict(parser.items('Activity')) + except configparser.NoSectionError as e: + raise BuildProcessError( + 'Error parsing metadata file. Exception message: %s', e + ) + + return attributes + + def check_bundle(bundle_name): + xo_file = zipfile.ZipFile(get_bundle_path(bundle_name)) + # Find the acitivity_file and return it + for filename in xo_file.namelist(): + if 'activity.info' in filename: + return xo_file.read(filename) + logger.info( + 'Bundle Check has failed. %s is not a valid bundle file ', bundle_name) + raise BuildProcessError( + 'Bundle Check has failed. %s is not a valid bundle file ', bundle_name) + + try: + activity_file = activity_file.deode() + except Exception as e: + raise BuildProcessError('Error decoding MeteData File. Exception Message: %s',e) + + \ No newline at end of file diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index 569d0c5..666ea3d 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -6,13 +6,30 @@ @celery.task(bind=True) def build_process(self, gh_json): try: - url = gh_json['repository']['clone_url'] - name = gh_json['repository']['name'] - tag = gh_json['release']['tag_name'] - + release = gh_json['release'] # TODO: work on cases (bundle attached or not) # and insert data into DB + # Invoke different build programs depending upon whether it's a source/asset release + if 'assets' in release and len(release['assets']) != 0: + handle_asset_release(gh_json) + else: + handle_source_release(gh_json) + + except BuildProcessError as e: + logger.exception("Error in activity building process") + return False + + logger.info('Activity building process finished !') + +def handle_source_release(gh_json): + logger.info('Building for a source release') + try: + url = gh_json['repository']['clone_url'] + name = gh_json['repository']['name'] + release = gh_json['release'] + tag = release['tag_name'] + build.clone_repo(url, name, tag) activity = build.get_activity_metadata(name) build.invoke_build(name) @@ -21,4 +38,33 @@ def build_process(self, gh_json): logger.exception("Error in activity building process") return False - logger.info('Activity building process finished succesfully!') +def handle_asset_release(gh_json): + logger.info('Building for a asset release') + try: + url = gh_json['repository']['clone_url'] + name = gh_json['repository']['name'] + release = gh_json['release'] + tag = release['tag_name'] + + bundle_name = build.check_and_download_assets(release['assets']) + build.invoke_asset_build(bundle_name) + + if bundle_name: + activity_file = utils.check_bundle(bundle_name) + if activity_file: + activity_file = activity_file.decode() + parser = utils.read_activity(activity_file,is_string=True) + print("Manifest Parsed .. OK") + # Check versions before invoking build + json_object = utils.get_activity_manifest(parser) + print("JSON Parsed .. OK") + print(json_object) + if is_a_new_release(json_object) is False: + # TODO - Inform Author about Failure + return "Failure" + print("Version check .. OK") + update_activity_record(json_object) + + except BuildProcessError as e: + logger.exception("Error in activity building process") + return False \ No newline at end of file diff --git a/aslo/default_settings.py b/aslo/default_settings.py index 01ec739..1ad2116 100644 --- a/aslo/default_settings.py +++ b/aslo/default_settings.py @@ -35,3 +35,5 @@ def env(variable, fallback_value=None): BUILD_CLONE_REPO = env('BUILD_CLONE_REPO', '/var/tmp/activities/') # Path where bundles are going to be stored BUILD_BUNDLE_DIR = env('BUILD_BUNDLE_DIR', '/srv/activities/') +# Temporary path to store bundles +TEMP_BUNDLE_DIR = env('TEMP_BUNDLE_DIR','/var/tmp/bundles/') \ No newline at end of file diff --git a/create_directories.sh b/create_directories.sh new file mode 100755 index 0000000..b509096 --- /dev/null +++ b/create_directories.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +current_user=$USER +echo "Creating directories ....." + +echo "Creating directory for storing downloaded repositories" +mkdir -p /tmp/activities +#chmod +w -R /tmp/activities +sudo setfacl -m u:$current_user:rwx /var/tmp/activities + +echo "Creating directory to store downloded bundled activities" + +sudo mkdir -p /opt/bundles +#sudo chmod +w -R /opt/bundles/ +sudo setfacl -m u:$current_user:rwx /srv/activities + +echo "Creating directory to store temporary bundles" +sudo mkdir -p /tmp/bundles + +sudo setfacl -m u:$current_user:rwx /tmp/bundles + +echo "Done " \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 21e36d7..6c4a307 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ flask_bootstrap celery redis pymongo +requests \ No newline at end of file diff --git a/start.sh b/start.sh old mode 100644 new mode 100755 index 5bdd6d4..c1ff623 --- a/start.sh +++ b/start.sh @@ -1,2 +1,2 @@ -. env/bin/activate +#. env/bin/activate exec honcho start From 5862a68e3be27e1cb50d41a9f18ad6ad2948b404 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Sun, 25 Jun 2017 16:11:12 +0530 Subject: [PATCH 03/57] Add partial models. Added Multi Asset Pipiline and Updated requirements --- aslo/api/build.py | 66 ++++++++++++++++++++++++++++------------- aslo/api/tasks.py | 16 ---------- aslo/models/MetaData.py | 19 ++++++++++++ requirements.txt | 3 +- 4 files changed, 67 insertions(+), 37 deletions(-) create mode 100644 aslo/models/MetaData.py diff --git a/aslo/api/build.py b/aslo/api/build.py index dac5735..c8c901b 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -17,20 +17,45 @@ def get_bundle_path(bundle_name): return os.path.join(app.config['BUILD_BUNDLE_DIR'], bundle_name) +def get_parser(activity_file, read_string=False): + parser = configparser.ConfigParser() + if read_string: + if len(parser.read_string(activity_file)) == 0: + raise BuildProcessError('Error parsing metadata file') + else: + return parser + else: + if len(parser.read(activity_file)) == 0: + raise BuildProcessError('Error parsing metadata file') + else: + return parser + + +def validate_metadata_attributes(parser, attributes): + MANDATORY_ATTRIBUTES = ['name', 'bundle_id', 'summary', 'license', + 'categories', 'icon', 'activity_version', 'repository', 'activity_version'] + return all(parser.has_option('Activity', attribute) for attribute in attributes) + + def check_and_download_assets(assets): + def check_asset(asset): + if asset_name_check(asset['name']): + return asset_manifest_check(asset['browser_download_url'], asset['name']) + return False + def download_asset(download_url, name): response = requests.get(download_url, stream=True) # Save with every block of 1024 bytes logger.info("Downloading File .. " + name) - with open(app.config['TEMP_BUNDLE_DIR'] + "/" + name, "wb") as handle: + with open(os.path.join(app.config['TEMP_BUNDLE_DIR'],name), "wb") as handle: for block in response.iter_content(chunk_size=1024): handle.write(block) return def check_info_file(name): logger.info("Checking For Activity.info") - xo_file = zipfile.ZipFile(app.config['TEMP_BUNDLE_DIR'] + "/" + name) + xo_file = zipfile.ZipFile(os.path.join(app.config['TEMP_BUNDLE_DIR'],name)) return any("activity.info" in filename for filename in xo_file.namelist()) def asset_name_check(asset_name): @@ -47,14 +72,20 @@ def asset_manifest_check(download_url, bundle_name): # Check if that bundle already exists then we don't continue # Return false if that particular bundle already exists if verify_bundle(bundle_name): - os.remove(app.config['TEMP_BUNDLE_DIR'] + "/" + bundle_name) + os.remove(os.path.join(app.config['TEMP_BUNDLE_DIR'],bundle_name)) raise BuildProcessError('File %s already exits' % bundle_name) else: - shutil.move(app.config['TEMP_BUNDLE_DIR'] + "/" + - bundle_name, app.config['BUILD_BUNDLE_DIR']) + shutil.move(os.path.join(app.config['TEMP_BUNDLE_DIR'], + bundle_name), app.config['BUILD_BUNDLE_DIR']) return bundle_name return False + for asset in assets: + bundle_name = check_asset(asset) + if bundle_name: + return bundle_name + raise BuildProcessError('No valid bundles were found in this asset release') + def clone_repo(url, name, tag): target_dir = app.config['BUILD_CLONE_REPO'] @@ -85,10 +116,7 @@ def metadata_file_exists(): return activity_file def parse_metadata_file(): - parser = configparser.ConfigParser() - if len(parser.read(activity_file)) == 0: - raise BuildProcessError('Error parsing metadata file') - + parser = get_parser(activity_file) try: attributes = dict(parser.items('Activity')) except configparser.NoSectionError as e: @@ -138,12 +166,9 @@ def clean(): clean() -def invoke_bundle_build(activity_file): +def invoke_asset_build(bundle_name): def parse_metadata_file(): - parser = configparser.ConfigParser() - if len(parser.read_string(activity_file)) == 0: - raise BuildProcessError('Error parsing metadata file') - + parser = get_parser(activity_file) try: attributes = dict(parser.items('Activity')) except configparser.NoSectionError as e: @@ -151,8 +176,8 @@ def parse_metadata_file(): 'Error parsing metadata file. Exception message: %s', e ) - return attributes - + return attributes + def check_bundle(bundle_name): xo_file = zipfile.ZipFile(get_bundle_path(bundle_name)) # Find the acitivity_file and return it @@ -165,8 +190,9 @@ def check_bundle(bundle_name): 'Bundle Check has failed. %s is not a valid bundle file ', bundle_name) try: - activity_file = activity_file.deode() + activity_file = check_bundle(bundle_name) + activity_file = activity_file.decode() + attributes = parse_metadata_file() except Exception as e: - raise BuildProcessError('Error decoding MeteData File. Exception Message: %s',e) - - \ No newline at end of file + raise BuildProcessError( + 'Error decoding MeteData File. Exception Message: %s', e) diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index 666ea3d..4fc3bf1 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -49,22 +49,6 @@ def handle_asset_release(gh_json): bundle_name = build.check_and_download_assets(release['assets']) build.invoke_asset_build(bundle_name) - if bundle_name: - activity_file = utils.check_bundle(bundle_name) - if activity_file: - activity_file = activity_file.decode() - parser = utils.read_activity(activity_file,is_string=True) - print("Manifest Parsed .. OK") - # Check versions before invoking build - json_object = utils.get_activity_manifest(parser) - print("JSON Parsed .. OK") - print(json_object) - if is_a_new_release(json_object) is False: - # TODO - Inform Author about Failure - return "Failure" - print("Version check .. OK") - update_activity_record(json_object) - except BuildProcessError as e: logger.exception("Error in activity building process") return False \ No newline at end of file diff --git a/aslo/models/MetaData.py b/aslo/models/MetaData.py new file mode 100644 index 0000000..4a87acb --- /dev/null +++ b/aslo/models/MetaData.py @@ -0,0 +1,19 @@ +from mongoengine import Document, StringField, DynamicEmbeddedDocument, EmbeddedDocumentListField + + +class Name(DynamicEmbeddedDocument): + # Empty class to avoid any empty object since we will have dynamic fields + pass + + +class Summary(DynamicEmbeddedDocument): + # Empty class to avoid any empty object since we will have dynamic fields + pass + + +class MetaData(Document): + name = EmbeddedDocumentListField(Name, required=True) + bundle_id = StringField(required=True) + summary = EmbeddedDocumentListField(Summary, required=True) + # Use GridFs to store images + icon = FileField() diff --git a/requirements.txt b/requirements.txt index 6c4a307..6a88c96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ flask_bootstrap celery redis pymongo -requests \ No newline at end of file +requests +mongoengine \ No newline at end of file From b041a5150bad8d69470bc65777c39cf91da7db71 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Tue, 27 Jun 2017 15:25:49 +0530 Subject: [PATCH 04/57] Fix activity parsing issues --- aslo/api/build.py | 18 +++++++++++++----- aslo/api/tasks.py | 5 +++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/aslo/api/build.py b/aslo/api/build.py index c8c901b..b81febd 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -7,7 +7,7 @@ from .exceptions import BuildProcessError import requests import zipfile - +import json def get_repo_location(name): return os.path.join(app.config['BUILD_CLONE_REPO'], name) @@ -16,12 +16,14 @@ def get_repo_location(name): def get_bundle_path(bundle_name): return os.path.join(app.config['BUILD_BUNDLE_DIR'], bundle_name) - + def get_parser(activity_file, read_string=False): parser = configparser.ConfigParser() if read_string: - if len(parser.read_string(activity_file)) == 0: - raise BuildProcessError('Error parsing metadata file') + try: + parser.read_string(activity_file) + except Exception as e: + raise BuildProcessError('Error parsing metadata file. Error : %s',e) else: return parser else: @@ -167,8 +169,12 @@ def clean(): def invoke_asset_build(bundle_name): + def remove_bundle(bundle_name): + logger.info("Removing Bundle : %s",bundle_name) + os.remove(get_bundle_path(bundle_name)) + def parse_metadata_file(): - parser = get_parser(activity_file) + parser = get_parser(activity_file,read_string=True) try: attributes = dict(parser.items('Activity')) except configparser.NoSectionError as e: @@ -194,5 +200,7 @@ def check_bundle(bundle_name): activity_file = activity_file.decode() attributes = parse_metadata_file() except Exception as e: + remove_bundle(bundle_name) raise BuildProcessError( 'Error decoding MeteData File. Exception Message: %s', e) + diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index 4fc3bf1..5c8c64e 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -33,7 +33,7 @@ def handle_source_release(gh_json): build.clone_repo(url, name, tag) activity = build.get_activity_metadata(name) build.invoke_build(name) - + logger.info(activity) except BuildProcessError as e: logger.exception("Error in activity building process") return False @@ -47,7 +47,8 @@ def handle_asset_release(gh_json): tag = release['tag_name'] bundle_name = build.check_and_download_assets(release['assets']) - build.invoke_asset_build(bundle_name) + activity = build.invoke_asset_build(bundle_name) + logger.info(activity) except BuildProcessError as e: logger.exception("Error in activity building process") From 7f95ac3364fc98ec8e6393eaeac03cd7aaf50a07 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Tue, 27 Jun 2017 16:59:23 +0530 Subject: [PATCH 05/57] Modify Schema. Algin Build Process --- aslo/api/build.py | 2 +- aslo/models/Activity.py | 35 +++++++++++++++++++++++++++++++++++ aslo/models/MetaData.py | 19 ------------------- 3 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 aslo/models/Activity.py delete mode 100644 aslo/models/MetaData.py diff --git a/aslo/api/build.py b/aslo/api/build.py index b81febd..87c0691 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -198,7 +198,7 @@ def check_bundle(bundle_name): try: activity_file = check_bundle(bundle_name) activity_file = activity_file.decode() - attributes = parse_metadata_file() + return parse_metadata_file() except Exception as e: remove_bundle(bundle_name) raise BuildProcessError( diff --git a/aslo/models/Activity.py b/aslo/models/Activity.py new file mode 100644 index 0000000..9138068 --- /dev/null +++ b/aslo/models/Activity.py @@ -0,0 +1,35 @@ +from mongoengine import Document, StringField, DynamicEmbeddedDocument, EmbeddedDocumentListField + + +class Name(DynamicEmbeddedDocument): + # Empty class to avoid any empty object since we will have dynamic fields + def __str__(self): + return self.to_json() + +class Summary(DynamicEmbeddedDocument): + # Empty class to avoid any empty object since we will have dynamic fields + def __str__(self): + return self.to_json() + +class Developer(DynamicEmbeddedDocument): + name = StringField(required=True) + email = StringField() + page = StringField() + + def __str__(self): + return self.to_json() + +class MetaData(Document): + name = EmbeddedDocumentListField(Name, required=True) + bundle_id = StringField(required=True) + summary = EmbeddedDocumentListField(Summary, required=True) + categories = StringField(default="") + activity_version = StringField(required=True) + repository = StringField(required=True) + developers = EmbeddedDocumentListField(Developer,required=True) + # Use GridFs to store images + #icon = FileField() + + + def __str__(self): + return self.to_json diff --git a/aslo/models/MetaData.py b/aslo/models/MetaData.py deleted file mode 100644 index 4a87acb..0000000 --- a/aslo/models/MetaData.py +++ /dev/null @@ -1,19 +0,0 @@ -from mongoengine import Document, StringField, DynamicEmbeddedDocument, EmbeddedDocumentListField - - -class Name(DynamicEmbeddedDocument): - # Empty class to avoid any empty object since we will have dynamic fields - pass - - -class Summary(DynamicEmbeddedDocument): - # Empty class to avoid any empty object since we will have dynamic fields - pass - - -class MetaData(Document): - name = EmbeddedDocumentListField(Name, required=True) - bundle_id = StringField(required=True) - summary = EmbeddedDocumentListField(Summary, required=True) - # Use GridFs to store images - icon = FileField() From 8c5f48587566a0d32b5979d9c880b45c2b175100 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Tue, 27 Jun 2017 17:55:43 +0530 Subject: [PATCH 06/57] Complete first draft of schema. Added a method to add releases --- aslo/models/Activity.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/aslo/models/Activity.py b/aslo/models/Activity.py index 9138068..836c7b2 100644 --- a/aslo/models/Activity.py +++ b/aslo/models/Activity.py @@ -1,4 +1,4 @@ -from mongoengine import Document, StringField, DynamicEmbeddedDocument, EmbeddedDocumentListField +from mongoengine import Document, StringField, ListField, DynamicEmbeddedDocument, EmbeddedDocumentListField, IntField, FloatField, URLField, BooleanField, DateTimeField, ReferenceField class Name(DynamicEmbeddedDocument): @@ -6,11 +6,13 @@ class Name(DynamicEmbeddedDocument): def __str__(self): return self.to_json() + class Summary(DynamicEmbeddedDocument): # Empty class to avoid any empty object since we will have dynamic fields def __str__(self): return self.to_json() + class Developer(DynamicEmbeddedDocument): name = StringField(required=True) email = StringField() @@ -19,17 +21,44 @@ class Developer(DynamicEmbeddedDocument): def __str__(self): return self.to_json() + +class Release(Document): + # We can aslo use StringField for storing versions + activity_version = IntField(required=True) + release_notes = StringField(required=True) + # Use custom class bound method to calculate min_sugar_version ? + min_sugar_version = FloatField(required=True) + # Also known as xo_url + download_url = URLField(required=True) + is_web = BooleanField(required=True, default=False) + is_gtk = BooleanField(required=True, default=False) + has_old_toolbars = BooleanField(required=False, default=False) + # Timestamp when the release was made + timestamp = DateTimeField(required=True) + + class MetaData(Document): name = EmbeddedDocumentListField(Name, required=True) bundle_id = StringField(required=True) summary = EmbeddedDocumentListField(Summary, required=True) + # We aslo use ListField for categories but then we need to join and slice it from the MetaData categories = StringField(default="") - activity_version = StringField(required=True) + #activity_version = StringField(required=True) repository = StringField(required=True) - developers = EmbeddedDocumentListField(Developer,required=True) + developers = EmbeddedDocumentListField(Developer, required=True) # Use GridFs to store images #icon = FileField() + latest_release = ReferenceField(Release) + previous_releases = ListField(ReferenceField(Release)) + + def add_release(self, release): + # If First release (No previous releases) then just copy the release + if self.previous_releases.empty(): + latest_release = release + else: + self.previous_releases.insert(self.latest_release) + self.latest_release = release def __str__(self): return self.to_json From 6db348a73981f1a69ae49027903993488affe00e Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Tue, 27 Jun 2017 23:42:33 +0530 Subject: [PATCH 07/57] Finish Schema design for Aslo. Focus on i18n and pofiles --- aslo/api/build.py | 28 ++++++++++++++++++++++++++++ aslo/models/Activity.py | 17 +++++++++++------ requirements.txt | 3 ++- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/aslo/api/build.py b/aslo/api/build.py index 87c0691..19d8fda 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -4,10 +4,38 @@ from flask import current_app as app from aslo.celery_app import logger from subprocess import call +from polib import pofile from .exceptions import BuildProcessError import requests import zipfile import json +import glob + +### Po crawling functionality inspired from Sam Parkinson's po crawl +### https://github.com/samdroid-apps/aslo/blob/master/bot/po_crawl.py + + +def translate(source_string,pofile_dir,source_language="en-US",target_language=None): + # If target language is specified we return only the intended version + if target_language is not None: + return get_translation_string(source_string,pofile_dir)[target_language] + # Else we return all translated strings + else: + return + + +def get_translation_string(source_string,source_language,pofile_dir,target_language=None): + translated_results = {} + matched_files = glob.glob(os.path.join(pofile_dir,"*.po")) + if len(matched_files) == 0: + raise BuildProcessError("Can't file po files for translation in the directory : %s",pofile_dir) + po_files = map(pofile,matched_files) + for i in po_files: + e = [e for e in i[0].translated_entries() if e.msgid == source_language] + if e: + print("{}".format(e.msgstr)) + tag = i[1].replace('_', '-') + def get_repo_location(name): return os.path.join(app.config['BUILD_CLONE_REPO'], name) diff --git a/aslo/models/Activity.py b/aslo/models/Activity.py index 836c7b2..916e3b9 100644 --- a/aslo/models/Activity.py +++ b/aslo/models/Activity.py @@ -1,4 +1,4 @@ -from mongoengine import Document, StringField, ListField, DynamicEmbeddedDocument, EmbeddedDocumentListField, IntField, FloatField, URLField, BooleanField, DateTimeField, ReferenceField +from mongoengine import Document, StringField, ListField, DynamicEmbeddedDocument, EmbeddedDocumentListField, IntField, FloatField, URLField, BooleanField, DateTimeField, ReferenceField,ValidationError class Name(DynamicEmbeddedDocument): @@ -36,6 +36,9 @@ class Release(Document): # Timestamp when the release was made timestamp = DateTimeField(required=True) + def __str__(self): + self.to_json() + class MetaData(Document): name = EmbeddedDocumentListField(Name, required=True) @@ -52,13 +55,15 @@ class MetaData(Document): previous_releases = ListField(ReferenceField(Release)) def add_release(self, release): - + if self.latest_release.activity_version >= release.activity_version: + # In that case delete the activity, since we can only reference data if we save it + release.delete() + raise ValidationError("New release activity version {} is less than the current version {}".format(self.latest_release.activity_version,release.activity_version)) # If First release (No previous releases) then just copy the release - if self.previous_releases.empty(): + if len(self.previous_releases) == 0: latest_release = release else: - self.previous_releases.insert(self.latest_release) + self.previous_releases.append(self.latest_release) self.latest_release = release - def __str__(self): - return self.to_json + diff --git a/requirements.txt b/requirements.txt index 6a88c96..e35c03d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ celery redis pymongo requests -mongoengine \ No newline at end of file +mongoengine +polib \ No newline at end of file From f281837b486e0e783f7a5aeba1b4425dedec9e07 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Wed, 28 Jun 2017 22:34:44 +0530 Subject: [PATCH 08/57] Add po crawler to get translation --- aslo/api/build.py | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/aslo/api/build.py b/aslo/api/build.py index 19d8fda..0dc57f6 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -9,33 +9,34 @@ import requests import zipfile import json -import glob +from glob import glob -### Po crawling functionality inspired from Sam Parkinson's po crawl -### https://github.com/samdroid-apps/aslo/blob/master/bot/po_crawl.py +def get_translations(activity_location): + po_files_location = os.path.join(activity_location,'po/') + translations = {} + + def get_language_code(filepath): + basename = os.path.basename(filepath) + return os.path.splitext(basename)[0] + + matched_files = glob(os.path.join(po_files_location,"*.po")) + if len(matched_files) == 0: + raise BuildProcessError("No po files found at location . %s",po_files_location) -def translate(source_string,pofile_dir,source_language="en-US",target_language=None): - # If target language is specified we return only the intended version - if target_language is not None: - return get_translation_string(source_string,pofile_dir)[target_language] - # Else we return all translated strings - else: - return + po_files = list(map(pofile,matched_files)) + language_codes = list(map(get_language_code,matched_files)) + # Intialize the dictionary + for language_code in language_codes: + translations[language_code] = {} -def get_translation_string(source_string,source_language,pofile_dir,target_language=None): - translated_results = {} - matched_files = glob.glob(os.path.join(pofile_dir,"*.po")) - if len(matched_files) == 0: - raise BuildProcessError("Can't file po files for translation in the directory : %s",pofile_dir) - po_files = map(pofile,matched_files) - for i in po_files: - e = [e for e in i[0].translated_entries() if e.msgid == source_language] - if e: - print("{}".format(e.msgstr)) - tag = i[1].replace('_', '-') - + for po_file,language_code in zip(po_files,language_codes): + for entry in po_file.translated_entries(): + #print(entry) + translations[language_code][entry.msgid] = entry.msgstr + + return translations def get_repo_location(name): return os.path.join(app.config['BUILD_CLONE_REPO'], name) From bce70ffc89f5f737a0e9f89e88aa576abc3df061 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Wed, 28 Jun 2017 22:35:16 +0530 Subject: [PATCH 09/57] Call translation code in build pipeline --- aslo/api/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index 5c8c64e..af5ec77 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -32,6 +32,8 @@ def handle_source_release(gh_json): build.clone_repo(url, name, tag) activity = build.get_activity_metadata(name) + # Get translations string invoking build, since we clean the repo afterwards + translation = build.get_translations(build.get_repo_location(name)) build.invoke_build(name) logger.info(activity) except BuildProcessError as e: From 62a45f62013f5f65cd3e90e2baf72e397486688c Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Thu, 29 Jun 2017 16:32:05 +0530 Subject: [PATCH 10/57] Add code to extract translations --- aslo/api/build.py | 30 ++++++++++++++++++++++++++++-- aslo/api/tasks.py | 5 ++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/aslo/api/build.py b/aslo/api/build.py index 0dc57f6..91f1995 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -10,7 +10,7 @@ import zipfile import json from glob import glob - +import uuid def get_translations(activity_location): po_files_location = os.path.join(activity_location,'po/') @@ -196,6 +196,32 @@ def clean(): store_bundle() clean() +def get_xo_translations(bundle_name): + logger.info("Opening translations") + def clean_up(extact_dir): + shutil.rmtree(extact_dir) + try: + logger.info(get_bundle_path(bundle_name)) + xo_archive = zipfile.ZipFile(get_bundle_path(bundle_name)) + # Create a random UUID to store the extracted material + random_uuid = uuid.uuid4().hex + # Create the folder with name as random UUID + extract_dir = os.path.join(app.config['TEMP_BUNDLE_DIR'],random_uuid) + os.mkdir(extract_dir) + logger.info(extract_dir) + # Find root_prefix for the activities usually it's Name.Activity + archive_root_prefix = os.path.commonpath(xo_archive.namelist()) + xo_archive.extractall(path=extract_dir) + translations = get_translations(os.path.join(extract_dir,archive_root_prefix)) + # Clean up + clean_up(extract_dir) + return translations + except Exception as e: + # If exception is cause due to FileExistError, probably that two uuids were same somehow then clean up + if e.__class__.__name__ is "FileExistsError": + clean_up(extract_dir) + raise BuildProcessError("Unable to open archive : %s. Error : %s ",bundle_name,e.__class__) + def invoke_asset_build(bundle_name): def remove_bundle(bundle_name): @@ -212,7 +238,7 @@ def parse_metadata_file(): ) return attributes - + def check_bundle(bundle_name): xo_file = zipfile.ZipFile(get_bundle_path(bundle_name)) # Find the acitivity_file and return it diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index af5ec77..228e0b1 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -33,9 +33,10 @@ def handle_source_release(gh_json): build.clone_repo(url, name, tag) activity = build.get_activity_metadata(name) # Get translations string invoking build, since we clean the repo afterwards - translation = build.get_translations(build.get_repo_location(name)) + translations = build.get_translations(build.get_repo_location(name)) build.invoke_build(name) logger.info(activity) + logger.info(translations["es"]) except BuildProcessError as e: logger.exception("Error in activity building process") return False @@ -50,6 +51,8 @@ def handle_asset_release(gh_json): bundle_name = build.check_and_download_assets(release['assets']) activity = build.invoke_asset_build(bundle_name) + translations = build.get_xo_translations(bundle_name) + logger.info(translations["es"]) logger.info(activity) except BuildProcessError as e: From f98dc344a08a50dc4867d901741c1ae67b9ae813 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Thu, 29 Jun 2017 19:30:16 +0530 Subject: [PATCH 11/57] Add part of the db interaction layer --- aslo/api/build.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/aslo/api/build.py b/aslo/api/build.py index 91f1995..8f83136 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -11,6 +11,8 @@ import json from glob import glob import uuid +from mongoengine import connect +from aslo.models.Activity import MetaData,Release,Developer,Summary,Name def get_translations(activity_location): po_files_location = os.path.join(activity_location,'po/') @@ -196,6 +198,26 @@ def clean(): store_bundle() clean() +def populate_database(activity,translations): + def translate_field(field_value,model_class): + results = [] + for language_code in translations: + if field_value in translations[language_code]: + obj =model_class() + obj[language_code] = field_value + results.append(obj) + return results + + connect(app.config['MONGO_DBNAME']) + try: + metadata = MetaData() + metadata.name.extend(translate_field(activity['name'],Name)) + metadata.summary.extend(translate_field(activity['summary'],Summary)) + + + except Exception as e: + raise BuildProcessError("Failed to insert data inside the DB. Error : %s",e) + def get_xo_translations(bundle_name): logger.info("Opening translations") def clean_up(extact_dir): From 5c1c6772cbc7be51e40c5492a9502ae73fcf92de Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Fri, 30 Jun 2017 23:20:21 +0530 Subject: [PATCH 12/57] Re-format as per pep8. Standars. Use FloatFields for storing versions. Introduce IMGUR credentials --- aslo/api/build.py | 79 +++++++++++++++++++++++----------------- aslo/api/tasks.py | 13 ++++--- aslo/default_settings.py | 6 ++- aslo/models/Activity.py | 11 +++--- 4 files changed, 62 insertions(+), 47 deletions(-) diff --git a/aslo/api/build.py b/aslo/api/build.py index 8f83136..c7d751d 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -12,34 +12,37 @@ from glob import glob import uuid from mongoengine import connect -from aslo.models.Activity import MetaData,Release,Developer,Summary,Name +from aslo.models.Activity import MetaData, Release, Developer, Summary, Name + def get_translations(activity_location): - po_files_location = os.path.join(activity_location,'po/') + po_files_location = os.path.join(activity_location, 'po/') translations = {} - + def get_language_code(filepath): basename = os.path.basename(filepath) return os.path.splitext(basename)[0] - - matched_files = glob(os.path.join(po_files_location,"*.po")) + + matched_files = glob(os.path.join(po_files_location, "*.po")) if len(matched_files) == 0: - raise BuildProcessError("No po files found at location . %s",po_files_location) + raise BuildProcessError( + "No po files found at location . %s", po_files_location) - po_files = list(map(pofile,matched_files)) - language_codes = list(map(get_language_code,matched_files)) + po_files = list(map(pofile, matched_files)) + language_codes = list(map(get_language_code, matched_files)) # Intialize the dictionary for language_code in language_codes: translations[language_code] = {} - for po_file,language_code in zip(po_files,language_codes): + for po_file, language_code in zip(po_files, language_codes): for entry in po_file.translated_entries(): - #print(entry) - translations[language_code][entry.msgid] = entry.msgstr + # print(entry) + translations[language_code][entry.msgid] = entry.msgstr return translations + def get_repo_location(name): return os.path.join(app.config['BUILD_CLONE_REPO'], name) @@ -47,14 +50,15 @@ def get_repo_location(name): def get_bundle_path(bundle_name): return os.path.join(app.config['BUILD_BUNDLE_DIR'], bundle_name) - + def get_parser(activity_file, read_string=False): parser = configparser.ConfigParser() if read_string: try: parser.read_string(activity_file) except Exception as e: - raise BuildProcessError('Error parsing metadata file. Error : %s',e) + raise BuildProcessError( + 'Error parsing metadata file. Error : %s', e) else: return parser else: @@ -81,14 +85,15 @@ def download_asset(download_url, name): response = requests.get(download_url, stream=True) # Save with every block of 1024 bytes logger.info("Downloading File .. " + name) - with open(os.path.join(app.config['TEMP_BUNDLE_DIR'],name), "wb") as handle: + with open(os.path.join(app.config['TEMP_BUNDLE_DIR'], name), "wb") as handle: for block in response.iter_content(chunk_size=1024): handle.write(block) return def check_info_file(name): logger.info("Checking For Activity.info") - xo_file = zipfile.ZipFile(os.path.join(app.config['TEMP_BUNDLE_DIR'],name)) + xo_file = zipfile.ZipFile(os.path.join( + app.config['TEMP_BUNDLE_DIR'], name)) return any("activity.info" in filename for filename in xo_file.namelist()) def asset_name_check(asset_name): @@ -105,11 +110,12 @@ def asset_manifest_check(download_url, bundle_name): # Check if that bundle already exists then we don't continue # Return false if that particular bundle already exists if verify_bundle(bundle_name): - os.remove(os.path.join(app.config['TEMP_BUNDLE_DIR'],bundle_name)) + os.remove(os.path.join( + app.config['TEMP_BUNDLE_DIR'], bundle_name)) raise BuildProcessError('File %s already exits' % bundle_name) else: shutil.move(os.path.join(app.config['TEMP_BUNDLE_DIR'], - bundle_name), app.config['BUILD_BUNDLE_DIR']) + bundle_name), app.config['BUILD_BUNDLE_DIR']) return bundle_name return False @@ -117,7 +123,8 @@ def asset_manifest_check(download_url, bundle_name): bundle_name = check_asset(asset) if bundle_name: return bundle_name - raise BuildProcessError('No valid bundles were found in this asset release') + raise BuildProcessError( + 'No valid bundles were found in this asset release') def clone_repo(url, name, tag): @@ -198,12 +205,13 @@ def clean(): store_bundle() clean() -def populate_database(activity,translations): - def translate_field(field_value,model_class): + +def populate_database(activity, translations): + def translate_field(field_value, model_class): results = [] for language_code in translations: if field_value in translations[language_code]: - obj =model_class() + obj = model_class() obj[language_code] = field_value results.append(obj) return results @@ -211,15 +219,17 @@ def translate_field(field_value,model_class): connect(app.config['MONGO_DBNAME']) try: metadata = MetaData() - metadata.name.extend(translate_field(activity['name'],Name)) - metadata.summary.extend(translate_field(activity['summary'],Summary)) - + metadata.name.extend(translate_field(activity['name'], Name)) + metadata.summary.extend(translate_field(activity['summary'], Summary)) except Exception as e: - raise BuildProcessError("Failed to insert data inside the DB. Error : %s",e) + raise BuildProcessError( + "Failed to insert data inside the DB. Error : %s", e) + def get_xo_translations(bundle_name): logger.info("Opening translations") + def clean_up(extact_dir): shutil.rmtree(extact_dir) try: @@ -228,13 +238,14 @@ def clean_up(extact_dir): # Create a random UUID to store the extracted material random_uuid = uuid.uuid4().hex # Create the folder with name as random UUID - extract_dir = os.path.join(app.config['TEMP_BUNDLE_DIR'],random_uuid) + extract_dir = os.path.join(app.config['TEMP_BUNDLE_DIR'], random_uuid) os.mkdir(extract_dir) logger.info(extract_dir) # Find root_prefix for the activities usually it's Name.Activity archive_root_prefix = os.path.commonpath(xo_archive.namelist()) xo_archive.extractall(path=extract_dir) - translations = get_translations(os.path.join(extract_dir,archive_root_prefix)) + translations = get_translations( + os.path.join(extract_dir, archive_root_prefix)) # Clean up clean_up(extract_dir) return translations @@ -242,16 +253,17 @@ def clean_up(extact_dir): # If exception is cause due to FileExistError, probably that two uuids were same somehow then clean up if e.__class__.__name__ is "FileExistsError": clean_up(extract_dir) - raise BuildProcessError("Unable to open archive : %s. Error : %s ",bundle_name,e.__class__) + raise BuildProcessError( + "Unable to open archive : %s. Error : %s ", bundle_name, e.__class__) def invoke_asset_build(bundle_name): def remove_bundle(bundle_name): - logger.info("Removing Bundle : %s",bundle_name) + logger.info("Removing Bundle : %s", bundle_name) os.remove(get_bundle_path(bundle_name)) - + def parse_metadata_file(): - parser = get_parser(activity_file,read_string=True) + parser = get_parser(activity_file, read_string=True) try: attributes = dict(parser.items('Activity')) except configparser.NoSectionError as e: @@ -260,7 +272,7 @@ def parse_metadata_file(): ) return attributes - + def check_bundle(bundle_name): xo_file = zipfile.ZipFile(get_bundle_path(bundle_name)) # Find the acitivity_file and return it @@ -277,7 +289,6 @@ def check_bundle(bundle_name): activity_file = activity_file.decode() return parse_metadata_file() except Exception as e: - remove_bundle(bundle_name) + remove_bundle(bundle_name) raise BuildProcessError( 'Error decoding MeteData File. Exception Message: %s', e) - diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index 228e0b1..e33c52c 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -11,7 +11,7 @@ def build_process(self, gh_json): # and insert data into DB # Invoke different build programs depending upon whether it's a source/asset release if 'assets' in release and len(release['assets']) != 0: - handle_asset_release(gh_json) + handle_asset_release(gh_json) else: handle_source_release(gh_json) @@ -29,10 +29,10 @@ def handle_source_release(gh_json): name = gh_json['repository']['name'] release = gh_json['release'] tag = release['tag_name'] - + build.clone_repo(url, name, tag) activity = build.get_activity_metadata(name) - # Get translations string invoking build, since we clean the repo afterwards + # Get translations string invoking build, since we clean the repo afterwards translations = build.get_translations(build.get_repo_location(name)) build.invoke_build(name) logger.info(activity) @@ -41,6 +41,7 @@ def handle_source_release(gh_json): logger.exception("Error in activity building process") return False + def handle_asset_release(gh_json): logger.info('Building for a asset release') try: @@ -48,13 +49,13 @@ def handle_asset_release(gh_json): name = gh_json['repository']['name'] release = gh_json['release'] tag = release['tag_name'] - + bundle_name = build.check_and_download_assets(release['assets']) activity = build.invoke_asset_build(bundle_name) translations = build.get_xo_translations(bundle_name) logger.info(translations["es"]) logger.info(activity) - + except BuildProcessError as e: logger.exception("Error in activity building process") - return False \ No newline at end of file + return False diff --git a/aslo/default_settings.py b/aslo/default_settings.py index 1ad2116..8025546 100644 --- a/aslo/default_settings.py +++ b/aslo/default_settings.py @@ -16,6 +16,10 @@ def env(variable, fallback_value=None): DEBUG = env('DEBUG', False) SECRET_KEY = env('SECRET_KEY', '') +# IMGUR API CREDENTIALS +IMGUR_CLIENT_ID = env('IMGUR_CLIENT_ID', '') +IMGUR_CLIENT_SECRET = env('IMGUR_CLIENT_SECRET', '') + # GITHUB WEBHOOK GITHUB_HOOK_SECRET = env('GITHUB_HOOK_SECRET', '') @@ -36,4 +40,4 @@ def env(variable, fallback_value=None): # Path where bundles are going to be stored BUILD_BUNDLE_DIR = env('BUILD_BUNDLE_DIR', '/srv/activities/') # Temporary path to store bundles -TEMP_BUNDLE_DIR = env('TEMP_BUNDLE_DIR','/var/tmp/bundles/') \ No newline at end of file +TEMP_BUNDLE_DIR = env('TEMP_BUNDLE_DIR', '/var/tmp/bundles/') diff --git a/aslo/models/Activity.py b/aslo/models/Activity.py index 916e3b9..87a01fa 100644 --- a/aslo/models/Activity.py +++ b/aslo/models/Activity.py @@ -1,4 +1,4 @@ -from mongoengine import Document, StringField, ListField, DynamicEmbeddedDocument, EmbeddedDocumentListField, IntField, FloatField, URLField, BooleanField, DateTimeField, ReferenceField,ValidationError +from mongoengine import Document, StringField, ListField, DynamicEmbeddedDocument, EmbeddedDocumentListField, IntField, FloatField, URLField, BooleanField, DateTimeField, ReferenceField, ValidationError class Name(DynamicEmbeddedDocument): @@ -24,7 +24,7 @@ def __str__(self): class Release(Document): # We can aslo use StringField for storing versions - activity_version = IntField(required=True) + activity_version = FloatField(required=True) release_notes = StringField(required=True) # Use custom class bound method to calculate min_sugar_version ? min_sugar_version = FloatField(required=True) @@ -56,14 +56,13 @@ class MetaData(Document): def add_release(self, release): if self.latest_release.activity_version >= release.activity_version: - # In that case delete the activity, since we can only reference data if we save it + # In that case delete the activity, since we can only reference data if we save it release.delete() - raise ValidationError("New release activity version {} is less than the current version {}".format(self.latest_release.activity_version,release.activity_version)) + raise ValidationError("New release activity version {} is less than the current version {}".format( + self.latest_release.activity_version, release.activity_version)) # If First release (No previous releases) then just copy the release if len(self.previous_releases) == 0: latest_release = release else: self.previous_releases.append(self.latest_release) self.latest_release = release - - From 53153bb18d7a85925b3bf0f53847ae29ca577fd5 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Sat, 1 Jul 2017 22:35:27 +0530 Subject: [PATCH 13/57] Add and initialze Imgur Client --- aslo/api/__init__.py | 5 +++-- requirements.txt | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/aslo/api/__init__.py b/aslo/api/__init__.py index a7b4fbb..2bcc1de 100644 --- a/aslo/api/__init__.py +++ b/aslo/api/__init__.py @@ -1,5 +1,6 @@ from flask import Blueprint - +from imgurpython import ImgurClient api = Blueprint('api', __name__) -from . import views, errors # noqa +imgurClient = ImgurClient(api.app.config['IMGUR_CLIENT_ID'],api.app.config['IMGUR_CLIENT_SECRET']) +from . import views, errors # noqa \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e35c03d..dbe53b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ redis pymongo requests mongoengine -polib \ No newline at end of file +polib +imgurpython \ No newline at end of file From c940f6638a1c5ccd71ac5b6eded7201c8454fda2 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Sat, 1 Jul 2017 23:05:58 +0530 Subject: [PATCH 14/57] Add simple docs --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..8508249 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +### aslo-v3 +This repo hosts the upcoming version of ASLO (Software centre) for SugarLabs + +### Instructions + +#### Install Dependenices +**Python 3.x** supported + +* **Docker (System)** - For running build tasks +* **Mongo (System)** - Primary Database Engine +* **Redis (System) ** - Broker for Background Tasks +* **Celery** - For running background tasks +* **Imgur** - For hosting Screenshots +* **Flower (Optional)** - Dashboard for Celery + +Install most of the app/python dependencies by running `pip install -r requirements.txt ` + +##### How to Run + +Before running set environment variables like `GITHUB_HOOK_SECRET`, `IMGUR_CLIENT_ID` , `IMGUR_CLIENT_SECRET` and others defined in the `default_settings.py` + +Run `./start.sh` to run the server + +Run `flower -A worker --port=5555` to run the dashboard server \ No newline at end of file From 09473c6882934ad867ba6e1ea16c39b7ea06b63f Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Sun, 2 Jul 2017 23:07:49 +0530 Subject: [PATCH 15/57] Add mock functions to upload images --- aslo/api/__init__.py | 2 +- aslo/api/build.py | 33 +++++++++++++++++++++++++++++++++ aslo/api/tasks.py | 1 + aslo/models/Activity.py | 3 ++- 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/aslo/api/__init__.py b/aslo/api/__init__.py index 2bcc1de..376223a 100644 --- a/aslo/api/__init__.py +++ b/aslo/api/__init__.py @@ -2,5 +2,5 @@ from imgurpython import ImgurClient api = Blueprint('api', __name__) -imgurClient = ImgurClient(api.app.config['IMGUR_CLIENT_ID'],api.app.config['IMGUR_CLIENT_SECRET']) +imgur_client = ImgurClient(api.app.config['IMGUR_CLIENT_ID'],api.app.config['IMGUR_CLIENT_SECRET']) from . import views, errors # noqa \ No newline at end of file diff --git a/aslo/api/build.py b/aslo/api/build.py index c7d751d..12c4a28 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -13,6 +13,7 @@ import uuid from mongoengine import connect from aslo.models.Activity import MetaData, Release, Developer, Summary, Name +from aslo.api import imgur_client def get_translations(activity_location): @@ -43,6 +44,27 @@ def get_language_code(filepath): return translations +def upload_activity_icon(icon_path): + """Uploads activity icon to Imgur. + + Args: + icon_path: Path where icon is stored + Returns: + Imgur Url where icon is hosted + Raises: + BuildProcessErorr: When icon can't be uploaded due to netowrk issues or file being not present + """ + if not (os.path.exists(icon_path) and os.path.isfile(icon_path)): + raise BuildProcessError("Cannot find icon at path %s", icon_path) + + try: + response = imgur_client.upload_from_path(icon_path) + return response['link'] + except Exception as e: + raise BuildProcessError( + "Something unexpected happened while uploading the icon. Exception Message %s", e) + + def get_repo_location(name): return os.path.join(app.config['BUILD_CLONE_REPO'], name) @@ -74,6 +96,17 @@ def validate_metadata_attributes(parser, attributes): return all(parser.has_option('Activity', attribute) for attribute in attributes) +def upload_screenshots(manifest_part=False, screenhost_folder_path=None, url_manifest=None): + # If screenshots are part of the manifest + # Space separated list of urls, then upload urls + # Returns a list of links + if manifest_part: + pass + else: + # A folder containing screenshots + pass + + def check_and_download_assets(assets): def check_asset(asset): diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index e33c52c..7868e89 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -32,6 +32,7 @@ def handle_source_release(gh_json): build.clone_repo(url, name, tag) activity = build.get_activity_metadata(name) + # Get translations string invoking build, since we clean the repo afterwards translations = build.get_translations(build.get_repo_location(name)) build.invoke_build(name) diff --git a/aslo/models/Activity.py b/aslo/models/Activity.py index 87a01fa..a81b05e 100644 --- a/aslo/models/Activity.py +++ b/aslo/models/Activity.py @@ -34,6 +34,7 @@ class Release(Document): is_gtk = BooleanField(required=True, default=False) has_old_toolbars = BooleanField(required=False, default=False) # Timestamp when the release was made + screenshots = ListField(URLField(required=False), required=False) timestamp = DateTimeField(required=True) def __str__(self): @@ -50,7 +51,7 @@ class MetaData(Document): repository = StringField(required=True) developers = EmbeddedDocumentListField(Developer, required=True) # Use GridFs to store images - #icon = FileField() + icon = URLField(required=True) latest_release = ReferenceField(Release) previous_releases = ListField(ReferenceField(Release)) From 782e8db4f60a9f0ec25869de848be31b5497b0e4 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Sun, 2 Jul 2017 23:27:48 +0530 Subject: [PATCH 16/57] Add Build Status to README. Add travis.yml for proper building --- .travis.yml | 17 +++++++++++++++++ README.md | 4 +++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..4d428d2 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,17 @@ +language: python + +python: + - "3.2" + - "3.3" + - "3.4" + - "3.5" + - "3.6" + +services: + - redis-server + - mongodb + - docker + +install: pip install -r requirements.txt + +script: bash start.sh \ No newline at end of file diff --git a/README.md b/README.md index 8508249..9636064 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Build Status](https://travis-ci.org/jatindhankhar/aslo-v3.svg?branch=master)](https://travis-ci.org/jatindhankhar/aslo-v3) + ### aslo-v3 This repo hosts the upcoming version of ASLO (Software centre) for SugarLabs @@ -8,7 +10,7 @@ This repo hosts the upcoming version of ASLO (Software centre) for SugarLabs * **Docker (System)** - For running build tasks * **Mongo (System)** - Primary Database Engine -* **Redis (System) ** - Broker for Background Tasks +* **Redis (System)** - Broker for Background Tasks * **Celery** - For running background tasks * **Imgur** - For hosting Screenshots * **Flower (Optional)** - Dashboard for Celery From 0a3399704ebd64b14596e8dbcaaa721dcaabf6b1 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Mon, 3 Jul 2017 23:32:56 +0530 Subject: [PATCH 17/57] Add encrypted environment variables to avoid failed build due to absence of valid keys --- .travis.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4d428d2..fa56f09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,19 @@ language: python - python: - - "3.2" - - "3.3" - - "3.4" - - "3.5" - - "3.6" - +- '3.2' +- '3.3' +- '3.4' +- '3.5' +- '3.6' services: - - redis-server - - mongodb - - docker - +- redis-server +- mongodb +- docker install: pip install -r requirements.txt +env: + global: + - secure: lZKvS/zFwWELzcBD8+Yum57GHFb3+Ltc9hibLpecwHe826+llMQCCH5lkZSEXWvc5rVgW28zQN3RcYnmW9iaePQXD6SIhQlrZej5kkwPvPXMxbqzRrN9iVaFgyTJiSVsy5onZUkwY6ixPMrBfcy6VbcnSnnJHhOYyMDw56Y4jam0Lv3rykoaHls+D+NHREA51pEuBLNL6Y2XrEIlIZePr6FtFT/Pwg4H1fvkAHpM8IaCEISUFairwrN6KiBHX1pF9JOkEanMA7weuR+z7VC7sKWLIsrJ9rGHXjFVkiZYtm6f6unH71EK3xRR/MeuEdQrPNDJsnikqZPRuCCV8YfFycjoOEoyDNU7y5FRS3ZXQOBaObPGb/QLlI4PpX96kQHfE3fXMMJG+NDkTGk8uFDB1Il7LYXZDYlNUk8/RE9azOLwy9sR9+ZIRd7O09ZO9vpptR/mjqrOVFY5NVm3zDlArDofpyJVVrmOaA//SY4oEy+jACS3sEUEiGXYpLkKJQb75avncnzVPxG6NYpcNSjeFMaF17+YLF5S2DfrQ5iZtATV+ysnYrNbgi0cEmIbh1bRwrizMqkQyjB/mzBF0S+JazOJDa125ZQEbNw9ulBMsOGQ4zaYBn+K/Yznb6Xwsariroabvczey2Jc4jvY9166BA1ieFmJjkgYFYXenT/JLSQ= + - secure: FoNQCY4CQDj7KQqD0ahwKVcyTjEC9kJytpAKxbgpwlfgPjUO4lNGVi0v7v09Dyr0yPeLlzNlqQmxt0NR80Hh+JNiRE+IVFeRHtLY3emH/6fPcFfCxRLLvgT9c3p4QeNHLz61SuqZ2/PHoWit20BHAosgFhCZ4rGqvOciPm6cJNbq4L2MA1PPzwpdFBuHWedNiMfF2FF1jnN3NFVbEurqqDGgDJz8rrYSu5HSu2qojSvPa2MF/nmAcDSdJw3rUeVytGlimytQ6rrf0ucxRVE7ynyWap4rBygDBzI/DZcs6zSwWhUCvQG66wwdGxnz9parAxGv7bZRUtL6IzE+QJnLY/bsjimusQ8WcE91onz74D/tX9pzJClO8woLyV75FHP/xi7I0UuETCWCRehwvkb/OS4657kBMj7xVApHOywJymRFKugKPiQKlPtbbKD+7Z4FczQ2qRfGHGnzLcv+UeQEO0R5fZuxBfG/BTXME73Qj/mzCq4LIrFezzQMSbCGQkBsmOvapzp8ClwEIwIVfQCCs+LrQlYvdf1PMxMTPXt2GCZoNUk8J3LG+U7anHJq0eHYRaW54iw61ipG031B+Jc/MoAgs0bU+SObnQk7rYyjEkOXcn+c018MLuZBwNnMGjo1ixc+vU4jEJt9S3qPDLQh0Knt3yI2idp7p5e1N1SCz80= + script: bash start.sh \ No newline at end of file From 633278fd6e334b9f654f559715ddbb21b7bb6d4e Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Tue, 4 Jul 2017 15:56:39 +0530 Subject: [PATCH 18/57] Add encrypted env file for Travis. Update instructions --- .env.enc | 2 ++ .travis.yml | 10 ++++------ README.md | 14 +++++++++++++- 3 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 .env.enc diff --git a/.env.enc b/.env.enc new file mode 100644 index 0000000..a6fdcd8 --- /dev/null +++ b/.env.enc @@ -0,0 +1,2 @@ +Bâÿ>Ø+ñªý`<¾¯‘Õf²•€5¬{ç +ðg€ZEÎj9d©€õ I LœDL vðªíÑdH×?;W¼S„Jsàt¼"0ý¥GQ£FÒ?‚¢¤Õ¦æŠcp{¢t€¬ IÊû óÖi¨¾Òˆågˆg%g®Fª,N° \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index fa56f09..9dc59e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,9 @@ services: - redis-server - mongodb - docker +before_install: +- openssl aes-256-cbc -K $encrypted_ad8f1517fc45_key -iv $encrypted_ad8f1517fc45_iv + -in .env.enc -out .env -d install: pip install -r requirements.txt +script: bash start.sh -env: - global: - - secure: lZKvS/zFwWELzcBD8+Yum57GHFb3+Ltc9hibLpecwHe826+llMQCCH5lkZSEXWvc5rVgW28zQN3RcYnmW9iaePQXD6SIhQlrZej5kkwPvPXMxbqzRrN9iVaFgyTJiSVsy5onZUkwY6ixPMrBfcy6VbcnSnnJHhOYyMDw56Y4jam0Lv3rykoaHls+D+NHREA51pEuBLNL6Y2XrEIlIZePr6FtFT/Pwg4H1fvkAHpM8IaCEISUFairwrN6KiBHX1pF9JOkEanMA7weuR+z7VC7sKWLIsrJ9rGHXjFVkiZYtm6f6unH71EK3xRR/MeuEdQrPNDJsnikqZPRuCCV8YfFycjoOEoyDNU7y5FRS3ZXQOBaObPGb/QLlI4PpX96kQHfE3fXMMJG+NDkTGk8uFDB1Il7LYXZDYlNUk8/RE9azOLwy9sR9+ZIRd7O09ZO9vpptR/mjqrOVFY5NVm3zDlArDofpyJVVrmOaA//SY4oEy+jACS3sEUEiGXYpLkKJQb75avncnzVPxG6NYpcNSjeFMaF17+YLF5S2DfrQ5iZtATV+ysnYrNbgi0cEmIbh1bRwrizMqkQyjB/mzBF0S+JazOJDa125ZQEbNw9ulBMsOGQ4zaYBn+K/Yznb6Xwsariroabvczey2Jc4jvY9166BA1ieFmJjkgYFYXenT/JLSQ= - - secure: FoNQCY4CQDj7KQqD0ahwKVcyTjEC9kJytpAKxbgpwlfgPjUO4lNGVi0v7v09Dyr0yPeLlzNlqQmxt0NR80Hh+JNiRE+IVFeRHtLY3emH/6fPcFfCxRLLvgT9c3p4QeNHLz61SuqZ2/PHoWit20BHAosgFhCZ4rGqvOciPm6cJNbq4L2MA1PPzwpdFBuHWedNiMfF2FF1jnN3NFVbEurqqDGgDJz8rrYSu5HSu2qojSvPa2MF/nmAcDSdJw3rUeVytGlimytQ6rrf0ucxRVE7ynyWap4rBygDBzI/DZcs6zSwWhUCvQG66wwdGxnz9parAxGv7bZRUtL6IzE+QJnLY/bsjimusQ8WcE91onz74D/tX9pzJClO8woLyV75FHP/xi7I0UuETCWCRehwvkb/OS4657kBMj7xVApHOywJymRFKugKPiQKlPtbbKD+7Z4FczQ2qRfGHGnzLcv+UeQEO0R5fZuxBfG/BTXME73Qj/mzCq4LIrFezzQMSbCGQkBsmOvapzp8ClwEIwIVfQCCs+LrQlYvdf1PMxMTPXt2GCZoNUk8J3LG+U7anHJq0eHYRaW54iw61ipG031B+Jc/MoAgs0bU+SObnQk7rYyjEkOXcn+c018MLuZBwNnMGjo1ixc+vU4jEJt9S3qPDLQh0Knt3yI2idp7p5e1N1SCz80= - -script: bash start.sh \ No newline at end of file diff --git a/README.md b/README.md index 9636064..67207f3 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,20 @@ Install most of the app/python dependencies by running `pip install -r requirem ##### How to Run -Before running set environment variables like `GITHUB_HOOK_SECRET`, `IMGUR_CLIENT_ID` , `IMGUR_CLIENT_SECRET` and others defined in the `default_settings.py` +`honcho` which is used in this project, looks for `.env` file, so define all the environment variables like `GITHUB_HOOK_SECRET`, `IMGUR_CLIENT_ID` , `IMGUR_CLIENT_SECRET` and others defined in the `default_settings.py` +**A sample example** + +``` bash +$ cat >.env < Date: Tue, 4 Jul 2017 16:00:13 +0530 Subject: [PATCH 19/57] Fix inherting issues by placing Imgur in the root init files and removing traces from other submodule init files --- aslo/__init__.py | 3 ++- aslo/api/__init__.py | 5 ++--- aslo/api/build.py | 22 +++++++++------------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/aslo/__init__.py b/aslo/__init__.py index 9b06388..3f1b31d 100644 --- a/aslo/__init__.py +++ b/aslo/__init__.py @@ -1,7 +1,7 @@ from flask import Flask from flask_bootstrap import Bootstrap import logging - +from imgurpython import ImgurClient logger = logging.getLogger(__name__) @@ -16,6 +16,7 @@ def init_app(): from .celery_app import init_celery init_celery(app) + imgur_client = ImgurClient(app.config['IMGUR_CLIENT_ID'],app.config['IMGUR_CLIENT_SECRET']) # blueprints from .web import web app.register_blueprint(web) diff --git a/aslo/api/__init__.py b/aslo/api/__init__.py index 376223a..a7b4fbb 100644 --- a/aslo/api/__init__.py +++ b/aslo/api/__init__.py @@ -1,6 +1,5 @@ from flask import Blueprint -from imgurpython import ImgurClient + api = Blueprint('api', __name__) -imgur_client = ImgurClient(api.app.config['IMGUR_CLIENT_ID'],api.app.config['IMGUR_CLIENT_SECRET']) -from . import views, errors # noqa \ No newline at end of file +from . import views, errors # noqa diff --git a/aslo/api/build.py b/aslo/api/build.py index 12c4a28..0620140 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -13,8 +13,6 @@ import uuid from mongoengine import connect from aslo.models.Activity import MetaData, Release, Developer, Summary, Name -from aslo.api import imgur_client - def get_translations(activity_location): po_files_location = os.path.join(activity_location, 'po/') @@ -43,7 +41,6 @@ def get_language_code(filepath): return translations - def upload_activity_icon(icon_path): """Uploads activity icon to Imgur. @@ -55,15 +52,14 @@ def upload_activity_icon(icon_path): BuildProcessErorr: When icon can't be uploaded due to netowrk issues or file being not present """ if not (os.path.exists(icon_path) and os.path.isfile(icon_path)): - raise BuildProcessError("Cannot find icon at path %s", icon_path) - + raise BuildProcessError("Cannot find icon at path %s",icon_path) + try: - response = imgur_client.upload_from_path(icon_path) + response = app.imgur_client.upload_from_path(icon_path) return response['link'] except Exception as e: - raise BuildProcessError( - "Something unexpected happened while uploading the icon. Exception Message %s", e) - + raise BuildProcessError("Something unexpected happened while uploading the icon. Exception Message %s",e) + def get_repo_location(name): return os.path.join(app.config['BUILD_CLONE_REPO'], name) @@ -96,17 +92,17 @@ def validate_metadata_attributes(parser, attributes): return all(parser.has_option('Activity', attribute) for attribute in attributes) -def upload_screenshots(manifest_part=False, screenhost_folder_path=None, url_manifest=None): +def upload_screenshots(manifest_part=False,screenhost_folder_path=None,url_manifest=None): # If screenshots are part of the manifest # Space separated list of urls, then upload urls # Returns a list of links if manifest_part: pass else: - # A folder containing screenshots + # A folder containing screenshots pass - - + + def check_and_download_assets(assets): def check_asset(asset): From 8664885aaca498d41e63abfa49b9bfc8374d4ba3 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Tue, 4 Jul 2017 16:17:57 +0530 Subject: [PATCH 20/57] Add timeout constraint for runnable server --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9dc59e4..364202e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,5 +13,6 @@ before_install: - openssl aes-256-cbc -K $encrypted_ad8f1517fc45_key -iv $encrypted_ad8f1517fc45_iv -in .env.enc -out .env -d install: pip install -r requirements.txt -script: bash start.sh +# Run server only for 50 seconds, some time given for Imgur Client to verify keys, ideally it takes less than 2 seconds to validate and boot up whole server +script: timeout -sHUP 0.5m bash start.sh From ebe5c05e4a34a5c6e8280327435c9b0e8eef0e55 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Tue, 4 Jul 2017 16:33:31 +0530 Subject: [PATCH 21/57] Using exit 0 trick to yield successful builds --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 364202e..16bb96e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,5 +14,5 @@ before_install: -in .env.enc -out .env -d install: pip install -r requirements.txt # Run server only for 50 seconds, some time given for Imgur Client to verify keys, ideally it takes less than 2 seconds to validate and boot up whole server -script: timeout -sHUP 0.5m bash start.sh +script: timeout -sHUP 0.5m bash start.sh; exit 0 From e55c513301bf01fc9f313ea4774f735aa33ca98b Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Tue, 4 Jul 2017 17:18:38 +0530 Subject: [PATCH 22/57] Add conditional check for successful builds --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 16bb96e..54f7986 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,5 +14,6 @@ before_install: -in .env.enc -out .env -d install: pip install -r requirements.txt # Run server only for 50 seconds, some time given for Imgur Client to verify keys, ideally it takes less than 2 seconds to validate and boot up whole server -script: timeout -sHUP 0.5m bash start.sh; exit 0 +# Timeout returns 124 if command timeouts without any problems +script: timeout -sHUP 0.5m bash start.sh; if [ "$?" -eq 124 ]; then exit 0 ; else exit 1 ; fi From 80dd31e282aa541ed2a1a43faef9feb340842240 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Tue, 4 Jul 2017 18:43:22 +0530 Subject: [PATCH 23/57] Remove Python 3.2 from build matrix --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 54f7986..9017216 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: -- '3.2' - '3.3' - '3.4' - '3.5' From b7fb7c5756178fe885797074c78babab8b5b60bc Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Tue, 4 Jul 2017 18:44:05 +0530 Subject: [PATCH 24/57] Refactor in Progress: Streamline Asset Image Uploads --- aslo/api/build.py | 118 ++++++++++++++++++++++++++++++++++++---------- aslo/api/tasks.py | 5 +- 2 files changed, 96 insertions(+), 27 deletions(-) diff --git a/aslo/api/build.py b/aslo/api/build.py index 0620140..4db91cf 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -14,6 +14,7 @@ from mongoengine import connect from aslo.models.Activity import MetaData, Release, Developer, Summary, Name + def get_translations(activity_location): po_files_location = os.path.join(activity_location, 'po/') translations = {} @@ -41,6 +42,38 @@ def get_language_code(filepath): return translations + +def upload_image_assets(attribute_dict, activity_location): + """Uploads all image assets to Imgur. + + Args: + attribute_dict: Dictionary containing activity.info attributes + repo_location: Location where repo/bundle is stored + Returns: + A dict having icon and screesnhots keys containing imgur urls + Raises: + BuildProcessErorr: When icon can't be uploaded due to netowrk issues or file being not present + """ + imgur_links = {"icon": "", "screenshots": []} + if "icon" not in attribute_dict: + raise BuildProcessError("Cannot find icons.") + icon_path = os.path.join( + activity_location, "activity", attribute_dict["icon"], ".svg") + imgur_links["icon"] = upload_activity_icon(icon_path) + + if "screenshots" in attribute_dict: + screenshot_links = upload_screenshots( + attribute_dict["screenshots"].split(" "), urls=True) + imgur_links["screenshots"].extend(screenshot_links) + + screenshot_path = os.path.join(activity_location, "screenshots") + if os.path.exists(screenshot_path): + screenshots = glob(os.path.join(screenshot_path, "*.*")) + imgur_links["screenshots"].extend(upload_screenshots(screenshots)) + + return imgur_links + + def upload_activity_icon(icon_path): """Uploads activity icon to Imgur. @@ -52,14 +85,19 @@ def upload_activity_icon(icon_path): BuildProcessErorr: When icon can't be uploaded due to netowrk issues or file being not present """ if not (os.path.exists(icon_path) and os.path.isfile(icon_path)): - raise BuildProcessError("Cannot find icon at path %s",icon_path) - + raise BuildProcessError("Cannot find icon at path %s", icon_path) + try: response = app.imgur_client.upload_from_path(icon_path) return response['link'] except Exception as e: - raise BuildProcessError("Something unexpected happened while uploading the icon. Exception Message %s",e) - + raise BuildProcessError( + "Something unexpected happened while uploading the icon. Exception Message %s", e) + + +def get_icon_path(icon_name, activity_name): + return os.path.join(get_repo_location(activity_name), "activity", icon_name, ".svg") + def get_repo_location(name): return os.path.join(app.config['BUILD_CLONE_REPO'], name) @@ -92,17 +130,22 @@ def validate_metadata_attributes(parser, attributes): return all(parser.has_option('Activity', attribute) for attribute in attributes) -def upload_screenshots(manifest_part=False,screenhost_folder_path=None,url_manifest=None): +def upload_screenshots(screenshots, urls=False): # If screenshots are part of the manifest # Space separated list of urls, then upload urls # Returns a list of links - if manifest_part: - pass + imgur_links = [] + if urls: + for screenshot in screenshots: + result = app.imgur_client.upload_from_url(screenshot) + imgur_links.append(result['link']) else: - # A folder containing screenshots - pass - - + for screenshot in screenshots: + result = app.imgur_client.upload_from_path(screenshot) + imgur_links.append(result['link']) + return imgur_links + + def check_and_download_assets(assets): def check_asset(asset): @@ -256,35 +299,58 @@ def translate_field(field_value, model_class): "Failed to insert data inside the DB. Error : %s", e) -def get_xo_translations(bundle_name): - logger.info("Opening translations") +def clean_up(extact_dir): + """ Delete extraction directory of bunlde + Args: + extract: Extraction path of bundle + Returns: + None + Raises: + None + """ + shutil.rmtree(extact_dir) + - def clean_up(extact_dir): - shutil.rmtree(extact_dir) +def extract_bundle(bundle_name): + """ Extracts bundle to random directory + Args: + bundle_name: Name of the bundle that needs to be extracted + Returns: + Path where bundle is extracted + Raises: + BuildProcessErorr: If file is corrupted and/or bundle cannot be extracted properly + """ + logger.info("Extracting %s ....", bundle_name) try: - logger.info(get_bundle_path(bundle_name)) xo_archive = zipfile.ZipFile(get_bundle_path(bundle_name)) # Create a random UUID to store the extracted material random_uuid = uuid.uuid4().hex # Create the folder with name as random UUID extract_dir = os.path.join(app.config['TEMP_BUNDLE_DIR'], random_uuid) os.mkdir(extract_dir) - logger.info(extract_dir) - # Find root_prefix for the activities usually it's Name.Activity archive_root_prefix = os.path.commonpath(xo_archive.namelist()) xo_archive.extractall(path=extract_dir) - translations = get_translations( - os.path.join(extract_dir, archive_root_prefix)) - # Clean up - clean_up(extract_dir) - return translations + extraction_path = os.path.join(extract_dir, archive_root_prefix) + except Exception as e: - # If exception is cause due to FileExistError, probably that two uuids were same somehow then clean up - if e.__class__.__name__ is "FileExistsError": - clean_up(extract_dir) + if e.__class__.__name__ is "FileExistsError": + clean_up(extract_dir) raise BuildProcessError( "Unable to open archive : %s. Error : %s ", bundle_name, e.__class__) +def get_xo_translations(extract_dir): + """ Wrapper function to crawl translations for a bundle + Args: + bundle_name: Directory where bunlde is extracted + Returns: + Dictionary containing translations + Raises: + None + """ + return get_translations(extract_dir) + + + def invoke_asset_build(bundle_name): def remove_bundle(bundle_name): diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index 7868e89..f13ac3c 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -32,9 +32,12 @@ def handle_source_release(gh_json): build.clone_repo(url, name, tag) activity = build.get_activity_metadata(name) - + + # TODO: Couple out clean repo so we can avoid uploading screenshots for yet to fail builds + # Upload icons # Get translations string invoking build, since we clean the repo afterwards translations = build.get_translations(build.get_repo_location(name)) + imgur_links = build.upload_image_assets(activity,build.get_repo_location(name)) build.invoke_build(name) logger.info(activity) logger.info(translations["es"]) From d9e6f7b9db3b720d887e99c16eb9c7dc30e8e27a Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Tue, 4 Jul 2017 23:22:53 +0530 Subject: [PATCH 25/57] Refactor asset processing code. Imgur doesn't allow svg uploads :sad: . Converting them to base64 instead --- aslo/__init__.py | 2 - aslo/api/build.py | 82 +++++++++++++++++++++++------------------ aslo/api/tasks.py | 5 ++- aslo/models/Activity.py | 4 +- 4 files changed, 52 insertions(+), 41 deletions(-) diff --git a/aslo/__init__.py b/aslo/__init__.py index 3f1b31d..754fd6c 100644 --- a/aslo/__init__.py +++ b/aslo/__init__.py @@ -1,7 +1,6 @@ from flask import Flask from flask_bootstrap import Bootstrap import logging -from imgurpython import ImgurClient logger = logging.getLogger(__name__) @@ -16,7 +15,6 @@ def init_app(): from .celery_app import init_celery init_celery(app) - imgur_client = ImgurClient(app.config['IMGUR_CLIENT_ID'],app.config['IMGUR_CLIENT_SECRET']) # blueprints from .web import web app.register_blueprint(web) diff --git a/aslo/api/build.py b/aslo/api/build.py index 4db91cf..af8fdc1 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -1,6 +1,7 @@ import os import shutil import configparser +import base64 from flask import current_app as app from aslo.celery_app import logger from subprocess import call @@ -13,6 +14,7 @@ import uuid from mongoengine import connect from aslo.models.Activity import MetaData, Release, Developer, Summary, Name +from imgurpython import ImgurClient def get_translations(activity_location): @@ -43,7 +45,7 @@ def get_language_code(filepath): return translations -def upload_image_assets(attribute_dict, activity_location): +def process_image_assets(attribute_dict, activity_location): """Uploads all image assets to Imgur. Args: @@ -54,53 +56,60 @@ def upload_image_assets(attribute_dict, activity_location): Raises: BuildProcessErorr: When icon can't be uploaded due to netowrk issues or file being not present """ - imgur_links = {"icon": "", "screenshots": []} + imgur_client = ImgurClient( + app.config['IMGUR_CLIENT_ID'], app.config['IMGUR_CLIENT_SECRET']) + processed_images = {"icon": "", "screenshots": []} if "icon" not in attribute_dict: raise BuildProcessError("Cannot find icons.") icon_path = os.path.join( - activity_location, "activity", attribute_dict["icon"], ".svg") - imgur_links["icon"] = upload_activity_icon(icon_path) + activity_location, "activity", attribute_dict["icon"] + ".svg") + processed_images["icon"] = convert_activity_icon(icon_path) if "screenshots" in attribute_dict: screenshot_links = upload_screenshots( attribute_dict["screenshots"].split(" "), urls=True) - imgur_links["screenshots"].extend(screenshot_links) + processed_images["screenshots"].extend(screenshot_links) screenshot_path = os.path.join(activity_location, "screenshots") if os.path.exists(screenshot_path): screenshots = glob(os.path.join(screenshot_path, "*.*")) - imgur_links["screenshots"].extend(upload_screenshots(screenshots)) + processed_images["screenshots"].extend(upload_screenshots(screenshots)) - return imgur_links + return processed_images -def upload_activity_icon(icon_path): - """Uploads activity icon to Imgur. +def convert_activity_icon(icon_path): + """Convert activity icon to base64 encoded strign. Args: icon_path: Path where icon is stored Returns: - Imgur Url where icon is hosted + Base64 encoded version of SVG Icon Raises: - BuildProcessErorr: When icon can't be uploaded due to netowrk issues or file being not present + BuildProcessErorr: When icon can't be converted or not found """ + imgur_client = ImgurClient( + app.config['IMGUR_CLIENT_ID'], app.config['IMGUR_CLIENT_SECRET']) if not (os.path.exists(icon_path) and os.path.isfile(icon_path)): raise BuildProcessError("Cannot find icon at path %s", icon_path) try: - response = app.imgur_client.upload_from_path(icon_path) - return response['link'] + logger.info("Icon path is %s", icon_path) + with open(icon_path, "rb") as f: + img_data = f.read() + encoded_string = base64.b64encode(img_data).decode() + return encoded_string except Exception as e: raise BuildProcessError( - "Something unexpected happened while uploading the icon. Exception Message %s", e) + "Something unexpected happened while converting the icon. Exception Message %s", e) def get_icon_path(icon_name, activity_name): - return os.path.join(get_repo_location(activity_name), "activity", icon_name, ".svg") + return os.path.join(get_repo_location(activity_name), "activity", icon_name + ".svg") def get_repo_location(name): - return os.path.join(app.config['BUILD_CLONE_REPO'], name) + return os.path.join(app.cnfig['BUILD_CLONE_REPO'], name) def get_bundle_path(bundle_name): @@ -134,14 +143,16 @@ def upload_screenshots(screenshots, urls=False): # If screenshots are part of the manifest # Space separated list of urls, then upload urls # Returns a list of links + imgur_client = ImgurClient( + app.config['IMGUR_CLIENT_ID'], app.config['IMGUR_CLIENT_SECRET']) imgur_links = [] if urls: for screenshot in screenshots: - result = app.imgur_client.upload_from_url(screenshot) + result = imgur_client.upload_from_url(screenshot) imgur_links.append(result['link']) else: for screenshot in screenshots: - result = app.imgur_client.upload_from_path(screenshot) + result = imgur_client.upload_from_path(screenshot) imgur_links.append(result['link']) return imgur_links @@ -278,7 +289,7 @@ def clean(): clean() -def populate_database(activity, translations): +def populate_database(activity, translations, imgur_links): def translate_field(field_value, model_class): results = [] for language_code in translations: @@ -300,19 +311,19 @@ def translate_field(field_value, model_class): def clean_up(extact_dir): - """ Delete extraction directory of bunlde - Args: - extract: Extraction path of bundle - Returns: - None - Raises: - None - """ - shutil.rmtree(extact_dir) + """ Delete extraction directory of bunlde + Args: + extract: Extraction path of bundle + Returns: + None + Raises: + None + """ + shutil.rmtree(extact_dir) def extract_bundle(bundle_name): - """ Extracts bundle to random directory + """ Extracts bundle to random directory Args: bundle_name: Name of the bundle that needs to be extracted Returns: @@ -331,15 +342,16 @@ def extract_bundle(bundle_name): archive_root_prefix = os.path.commonpath(xo_archive.namelist()) xo_archive.extractall(path=extract_dir) extraction_path = os.path.join(extract_dir, archive_root_prefix) - + return extraction_path except Exception as e: - if e.__class__.__name__ is "FileExistsError": - clean_up(extract_dir) + if e.__class__.__name__ is "FileExistsError": + clean_up(extract_dir) raise BuildProcessError( "Unable to open archive : %s. Error : %s ", bundle_name, e.__class__) + def get_xo_translations(extract_dir): - """ Wrapper function to crawl translations for a bundle + """ Wrapper function to crawl translations for a bundle Args: bundle_name: Directory where bunlde is extracted Returns: @@ -348,8 +360,6 @@ def get_xo_translations(extract_dir): None """ return get_translations(extract_dir) - - def invoke_asset_build(bundle_name): @@ -386,4 +396,4 @@ def check_bundle(bundle_name): except Exception as e: remove_bundle(bundle_name) raise BuildProcessError( - 'Error decoding MeteData File. Exception Message: %s', e) + 'Error decoding MetaData File. Exception Message: %s', e) diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index f13ac3c..8bf05ba 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -56,7 +56,10 @@ def handle_asset_release(gh_json): bundle_name = build.check_and_download_assets(release['assets']) activity = build.invoke_asset_build(bundle_name) - translations = build.get_xo_translations(bundle_name) + extracted_bundle = build.extract_bundle(bundle_name) + imgur_links = build.upload_image_assets(activity,extracted_bundle) + translations = build.get_xo_translations(extracted_bundle) + logger.info(translations["es"]) logger.info(activity) diff --git a/aslo/models/Activity.py b/aslo/models/Activity.py index a81b05e..f572fc3 100644 --- a/aslo/models/Activity.py +++ b/aslo/models/Activity.py @@ -50,8 +50,8 @@ class MetaData(Document): #activity_version = StringField(required=True) repository = StringField(required=True) developers = EmbeddedDocumentListField(Developer, required=True) - # Use GridFs to store images - icon = URLField(required=True) + # Base64 encoded string to store images + icon = StringField(required=True) latest_release = ReferenceField(Release) previous_releases = ListField(ReferenceField(Release)) From 951fa21bde42f906c71c6d0674747210ad5c3230 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Wed, 5 Jul 2017 17:20:02 +0530 Subject: [PATCH 26/57] Complete Database integration. --- aslo/api/build.py | 147 +++++++++++++++++++++++++++++++++++++--- aslo/api/tasks.py | 33 ++++++--- aslo/models/Activity.py | 6 +- 3 files changed, 165 insertions(+), 21 deletions(-) diff --git a/aslo/api/build.py b/aslo/api/build.py index af8fdc1..ade8a01 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -2,6 +2,7 @@ import shutil import configparser import base64 +import datetime from flask import current_app as app from aslo.celery_app import logger from subprocess import call @@ -46,7 +47,7 @@ def get_language_code(filepath): def process_image_assets(attribute_dict, activity_location): - """Uploads all image assets to Imgur. + """Processes all image assets and uploads screenshots to Imgur. Args: attribute_dict: Dictionary containing activity.info attributes @@ -105,11 +106,21 @@ def convert_activity_icon(icon_path): def get_icon_path(icon_name, activity_name): + """ Returns icon path for an activity. + + Args: + icon_name: Name of the icon + activity_name: Name of the activity + Returns: + Icon path of the activity + Raises: + None + """ return os.path.join(get_repo_location(activity_name), "activity", icon_name + ".svg") def get_repo_location(name): - return os.path.join(app.cnfig['BUILD_CLONE_REPO'], name) + return os.path.join(app.config['BUILD_CLONE_REPO'], name) def get_bundle_path(bundle_name): @@ -140,9 +151,16 @@ def validate_metadata_attributes(parser, attributes): def upload_screenshots(screenshots, urls=False): - # If screenshots are part of the manifest - # Space separated list of urls, then upload urls - # Returns a list of links + """ Uploads screenshots to Imgur. + + Args: + screenshots: List containing screenshots path or urls + urls: Default - False. Determines whether screenshots are paths or urls + Returns: + Returns a list containing coressponding imgur links + Raises: + None + """ imgur_client = ImgurClient( app.config['IMGUR_CLIENT_ID'], app.config['IMGUR_CLIENT_SECRET']) imgur_links = [] @@ -227,6 +245,62 @@ def clone_repo(url, name, tag): raise BuildProcessError('[%s] command has failed' % ' '.join(cmd)) +def get_platform_versions(activity, repo_location): + logger.info("Applying heurisitc to determine platform versions") + + def is_web(): + if 'exec' in activity: + return activity['exec'] == 'sugar-activity-web' + return False # Fallback + + GTK3_IMPORT_TYPES = {'sugar3': 3, 'from gi.repository import Gtk': 3, + 'sugar.': 2, 'import pygtk': 2, 'pygtk.require': 2} + + def is_gtk3(): + setup_py_path = os.path.join(repo_location, 'setup.py') + all_files = os.listdir(repo_location) + try_paths = [setup_py_path] + all_files + + for path in try_paths: + if os.path.isfile(path): + with open(path) as f: + text = f.read() + for sign in GTK3_IMPORT_TYPES: + if sign in text: + version = GTK3_IMPORT_TYPES[sign] + return version == 3 + + # Fallback to assuming GTK3 + return True + + OLD_TOOLBAR_SIGNS = ['activity.ActivityToolbox', 'gtk.Toolbar'] + + def has_old_toolbars(): + for path in os.listdir(repo_location): + if os.path.isfile(path): + with open(path) as f: + text = f.read() + for sign in OLD_TOOLBAR_SIGNS: + if sign in text: + return True + return False + + def determine_min_sugar_version(is_gtk3, is_web, has_old_toolbars): + min_sugar_version = '0.100' if is_web else ( + '0.96' if is_gtk3 else ( + '0.86' if not has_old_toolbars else '0.82' + )) + return min_sugar_version + + platform_versions = {} + platform_versions['is_gtk3'] = is_gtk3() + platform_versions['is_web'] = is_web() + platform_versions['has_old_toolbars'] = has_old_toolbars() + platform_versions['min_sugar_version'] = determine_min_sugar_version(platform_versions['is_gtk3'], platform_versions['is_web'], platform_versions['has_old_toolbars']) + + return platform_versions + + def get_activity_metadata(name): def metadata_file_exists(): repo_dir = get_repo_location(name) @@ -253,6 +327,19 @@ def parse_metadata_file(): return parse_metadata_file() +def clean_repo(repo_name): + """ Deletes cloned repository. + + Args: + repo_name: Name of the repository + Returns: + None + Raises: + None + """ + shutil.rmtree(get_repo_location(repo_name)) + + def invoke_build(name): def store_bundle(): dist_dir = os.path.join(get_repo_location(name), 'dist') @@ -275,9 +362,6 @@ def store_bundle(): logger.info('Bundle succesfully stored at %s', stored_bundle) - def clean(): - shutil.rmtree(get_repo_location(name)) - volume = get_repo_location(name) + ':/activity' docker_image = app.config['BUILD_DOCKER_IMAGE'] docker_cmd = ['docker', 'run', '--rm', '-v', volume, docker_image] @@ -286,11 +370,32 @@ def clean(): raise BuildProcessError('Docker building process has failed') store_bundle() - clean() -def populate_database(activity, translations, imgur_links): +def populate_database(activity, translations, processed_images): + """ Populates MongoDB with MONGO_DBNAME as the target table/collection + + Args: + repo_name: Dictionary containing activity.info attributes + translations: Dictionary containing translations + processed_images: Dictionary processed icon and screenshots + Returns: + None + Raises: + BuildProccessError: When version numbers are not correct (less than the existing one in Database) or when Data cannot be inserted + """ + def translate_field(field_value, model_class): + """ Helper function which translates a field value and prepares an object with language code as keys + + Args: + field_value: Value of the String to be converted + model_class: Model class whose object needs to prepared for insertion + Returns: + Model class object populated with proper translations of the word if any + Raises: + None + """ results = [] for language_code in translations: if field_value in translations[language_code]: @@ -303,9 +408,31 @@ def translate_field(field_value, model_class): try: metadata = MetaData() metadata.name.extend(translate_field(activity['name'], Name)) + metadata.bundle_id = activity['bundle_id'] metadata.summary.extend(translate_field(activity['summary'], Summary)) + if 'categories' in activity: + metadata.categories = activity['categories'] + metadata.repository = activity['repository'] + metadata.icon = processed_images['icon'] + + release = Release() + release.activity_version = float(activity['activity_version']) + release.release_notes = activity['release_info']['release_time'] + release.download_url = "https://mock.org/download_url" + release.min_sugar_version = float(activity['platform_versions']['min_sugar_version']) + release.is_web = activity['platform_versions']['is_web'] + release.has_old_toolbars = activity['platform_versions']['has_old_toolbars'] + release.screenshots.extend(processed_images['screenshots']) + release_time = datetime.datetime.strptime( activity['release_info']['release_time'], "%Y-%m-%dT%H:%M:%SZ" ) + release.timestamp = release_time + release.save() + + metadata.add_release(release) + metadata.save() except Exception as e: + metadata.delete() + release.delete() raise BuildProcessError( "Failed to insert data inside the DB. Error : %s", e) diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index 8bf05ba..256a65c 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -32,14 +32,21 @@ def handle_source_release(gh_json): build.clone_repo(url, name, tag) activity = build.get_activity_metadata(name) - - # TODO: Couple out clean repo so we can avoid uploading screenshots for yet to fail builds - # Upload icons - # Get translations string invoking build, since we clean the repo afterwards - translations = build.get_translations(build.get_repo_location(name)) - imgur_links = build.upload_image_assets(activity,build.get_repo_location(name)) + # Insert/Override repository tag + activity['repository'] = url + repo_location = build.get_repo_location(name) build.invoke_build(name) + translations = build.get_translations(repo_location) + processed_images = build.process_image_assets(activity, repo_location) + activity['platform_versions'] = build.get_platform_versions( + activity, repo_location) + activity['release_info'] = {} + activity['release_info']['release_notes'] = gh_json['release']['body'] + activity['release_info']['release_time'] = gh_json['release']['published_at'] + build.clean_repo(name) + build.populate_database(activity, translations, processed_images) logger.info(activity) + logger.info(processed_images) logger.info(translations["es"]) except BuildProcessError as e: logger.exception("Error in activity building process") @@ -56,10 +63,20 @@ def handle_asset_release(gh_json): bundle_name = build.check_and_download_assets(release['assets']) activity = build.invoke_asset_build(bundle_name) + # Insert/Override repository tag + activity['repository'] = url extracted_bundle = build.extract_bundle(bundle_name) - imgur_links = build.upload_image_assets(activity,extracted_bundle) - translations = build.get_xo_translations(extracted_bundle) + processed_images = build.process_image_assets( + activity, extracted_bundle) + activity['platform_versions'] = build.get_platform_versions( + activity, extracted_bundle) + activity['release_info'] = {} + activity['release_info']['release_notes'] = gh_json['release']['body'] + activity['release_info']['release_time'] = gh_json['release']['published_at'] + translations = build.get_xo_translations(extracted_bundle) + build.populate_database(activity, translations, processed_images) + logger.info(processed_images) logger.info(translations["es"]) logger.info(activity) diff --git a/aslo/models/Activity.py b/aslo/models/Activity.py index f572fc3..f5299d4 100644 --- a/aslo/models/Activity.py +++ b/aslo/models/Activity.py @@ -49,20 +49,20 @@ class MetaData(Document): categories = StringField(default="") #activity_version = StringField(required=True) repository = StringField(required=True) - developers = EmbeddedDocumentListField(Developer, required=True) + developers = EmbeddedDocumentListField(Developer, required=False) # Base64 encoded string to store images icon = StringField(required=True) latest_release = ReferenceField(Release) previous_releases = ListField(ReferenceField(Release)) def add_release(self, release): - if self.latest_release.activity_version >= release.activity_version: + if self.latest_release is not None and self.latest_release.activity_version >= release.activity_version: # In that case delete the activity, since we can only reference data if we save it release.delete() raise ValidationError("New release activity version {} is less than the current version {}".format( self.latest_release.activity_version, release.activity_version)) # If First release (No previous releases) then just copy the release - if len(self.previous_releases) == 0: + if self.latest_release is not None and len(self.previous_releases) == 0: latest_release = release else: self.previous_releases.append(self.latest_release) From 965128d4669be6a598f9694187c8a92a92c52370 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Wed, 5 Jul 2017 19:17:27 +0530 Subject: [PATCH 27/57] Fix DB insertion error by incoporating @scanterog's version of add release. Test with both releases --- aslo/api/build.py | 25 +++++++++++++++---------- aslo/models/Activity.py | 29 ++++++++++++++++++----------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/aslo/api/build.py b/aslo/api/build.py index ade8a01..4cb5dfe 100644 --- a/aslo/api/build.py +++ b/aslo/api/build.py @@ -264,9 +264,9 @@ def is_gtk3(): for path in try_paths: if os.path.isfile(path): with open(path) as f: - text = f.read() - for sign in GTK3_IMPORT_TYPES: - if sign in text: + text = f.read() + for sign in GTK3_IMPORT_TYPES: + if sign in text: version = GTK3_IMPORT_TYPES[sign] return version == 3 @@ -279,7 +279,7 @@ def has_old_toolbars(): for path in os.listdir(repo_location): if os.path.isfile(path): with open(path) as f: - text = f.read() + text = f.read() for sign in OLD_TOOLBAR_SIGNS: if sign in text: return True @@ -296,7 +296,8 @@ def determine_min_sugar_version(is_gtk3, is_web, has_old_toolbars): platform_versions['is_gtk3'] = is_gtk3() platform_versions['is_web'] = is_web() platform_versions['has_old_toolbars'] = has_old_toolbars() - platform_versions['min_sugar_version'] = determine_min_sugar_version(platform_versions['is_gtk3'], platform_versions['is_web'], platform_versions['has_old_toolbars']) + platform_versions['min_sugar_version'] = determine_min_sugar_version( + platform_versions['is_gtk3'], platform_versions['is_web'], platform_versions['has_old_toolbars']) return platform_versions @@ -409,7 +410,9 @@ def translate_field(field_value, model_class): metadata = MetaData() metadata.name.extend(translate_field(activity['name'], Name)) metadata.bundle_id = activity['bundle_id'] - metadata.summary.extend(translate_field(activity['summary'], Summary)) + if 'summary' in activity: + summaries = translate_field(activity['summary'], Summary) + metadata.summary.extend(summaries) if 'categories' in activity: metadata.categories = activity['categories'] metadata.repository = activity['repository'] @@ -417,16 +420,18 @@ def translate_field(field_value, model_class): release = Release() release.activity_version = float(activity['activity_version']) - release.release_notes = activity['release_info']['release_time'] + release.release_notes = activity['release_info']['release_notes'] release.download_url = "https://mock.org/download_url" - release.min_sugar_version = float(activity['platform_versions']['min_sugar_version']) + release.min_sugar_version = float( + activity['platform_versions']['min_sugar_version']) release.is_web = activity['platform_versions']['is_web'] release.has_old_toolbars = activity['platform_versions']['has_old_toolbars'] release.screenshots.extend(processed_images['screenshots']) - release_time = datetime.datetime.strptime( activity['release_info']['release_time'], "%Y-%m-%dT%H:%M:%SZ" ) + release_time = datetime.datetime.strptime( + activity['release_info']['release_time'], "%Y-%m-%dT%H:%M:%SZ") release.timestamp = release_time - release.save() + release.save() metadata.add_release(release) metadata.save() diff --git a/aslo/models/Activity.py b/aslo/models/Activity.py index f5299d4..7d93706 100644 --- a/aslo/models/Activity.py +++ b/aslo/models/Activity.py @@ -44,7 +44,7 @@ def __str__(self): class MetaData(Document): name = EmbeddedDocumentListField(Name, required=True) bundle_id = StringField(required=True) - summary = EmbeddedDocumentListField(Summary, required=True) + summary = EmbeddedDocumentListField(Summary, required=False, default=[]) # We aslo use ListField for categories but then we need to join and slice it from the MetaData categories = StringField(default="") #activity_version = StringField(required=True) @@ -56,14 +56,21 @@ class MetaData(Document): previous_releases = ListField(ReferenceField(Release)) def add_release(self, release): - if self.latest_release is not None and self.latest_release.activity_version >= release.activity_version: - # In that case delete the activity, since we can only reference data if we save it - release.delete() - raise ValidationError("New release activity version {} is less than the current version {}".format( - self.latest_release.activity_version, release.activity_version)) - # If First release (No previous releases) then just copy the release - if self.latest_release is not None and len(self.previous_releases) == 0: - latest_release = release - else: - self.previous_releases.append(self.latest_release) + # first release + if not self.latest_release and len(self.previous_releases) == 0: self.latest_release = release + return + + if self.latest_release.activity_version >= release.activity_version: + release.delete() + raise ValidationError( + 'New activity release version {} is less or equal than the ' + 'current version {}' + .format( + release.activity_version, + self.latest_release.activity_version + ) + ) + + self.previous_releases.append(self.latest_release) + self.latest_release = release From 77bd6ec08c6573a8dc642d44597e5569a185798f Mon Sep 17 00:00:00 2001 From: Samuel Cantero Date: Thu, 6 Jul 2017 21:37:45 -0400 Subject: [PATCH 28/57] Simplify code for managing bundle and source code releases * Improve workflow for both release processes. It reduces special handling for both cases. * Improve error handling. * Add GitHub client in order to get contributors of the repo. --- aslo/__init__.py | 3 +- aslo/api/build.py | 531 --------------------------------------- aslo/api/exceptions.py | 6 +- aslo/api/gh.py | 31 +++ aslo/api/i18n.py | 35 +++ aslo/api/release.py | 351 ++++++++++++++++++++++++++ aslo/api/tasks.py | 87 +------ aslo/api/views.py | 6 +- aslo/default_settings.py | 2 + aslo/models/Activity.py | 101 +++++--- requirements.txt | 4 +- 11 files changed, 501 insertions(+), 656 deletions(-) delete mode 100644 aslo/api/build.py create mode 100644 aslo/api/gh.py create mode 100644 aslo/api/i18n.py create mode 100644 aslo/api/release.py diff --git a/aslo/__init__.py b/aslo/__init__.py index 754fd6c..760ac13 100644 --- a/aslo/__init__.py +++ b/aslo/__init__.py @@ -1,6 +1,7 @@ from flask import Flask from flask_bootstrap import Bootstrap import logging + logger = logging.getLogger(__name__) @@ -21,7 +22,7 @@ def init_app(): from .api import api app.register_blueprint(api, url_prefix='/api') - + # logging logger.setLevel(logging.INFO) fmt = logging.Formatter('[%(asctime)s] %(levelname).3s %(message)s') diff --git a/aslo/api/build.py b/aslo/api/build.py deleted file mode 100644 index 4cb5dfe..0000000 --- a/aslo/api/build.py +++ /dev/null @@ -1,531 +0,0 @@ -import os -import shutil -import configparser -import base64 -import datetime -from flask import current_app as app -from aslo.celery_app import logger -from subprocess import call -from polib import pofile -from .exceptions import BuildProcessError -import requests -import zipfile -import json -from glob import glob -import uuid -from mongoengine import connect -from aslo.models.Activity import MetaData, Release, Developer, Summary, Name -from imgurpython import ImgurClient - - -def get_translations(activity_location): - po_files_location = os.path.join(activity_location, 'po/') - translations = {} - - def get_language_code(filepath): - basename = os.path.basename(filepath) - return os.path.splitext(basename)[0] - - matched_files = glob(os.path.join(po_files_location, "*.po")) - if len(matched_files) == 0: - raise BuildProcessError( - "No po files found at location . %s", po_files_location) - - po_files = list(map(pofile, matched_files)) - language_codes = list(map(get_language_code, matched_files)) - - # Intialize the dictionary - for language_code in language_codes: - translations[language_code] = {} - - for po_file, language_code in zip(po_files, language_codes): - for entry in po_file.translated_entries(): - # print(entry) - translations[language_code][entry.msgid] = entry.msgstr - - return translations - - -def process_image_assets(attribute_dict, activity_location): - """Processes all image assets and uploads screenshots to Imgur. - - Args: - attribute_dict: Dictionary containing activity.info attributes - repo_location: Location where repo/bundle is stored - Returns: - A dict having icon and screesnhots keys containing imgur urls - Raises: - BuildProcessErorr: When icon can't be uploaded due to netowrk issues or file being not present - """ - imgur_client = ImgurClient( - app.config['IMGUR_CLIENT_ID'], app.config['IMGUR_CLIENT_SECRET']) - processed_images = {"icon": "", "screenshots": []} - if "icon" not in attribute_dict: - raise BuildProcessError("Cannot find icons.") - icon_path = os.path.join( - activity_location, "activity", attribute_dict["icon"] + ".svg") - processed_images["icon"] = convert_activity_icon(icon_path) - - if "screenshots" in attribute_dict: - screenshot_links = upload_screenshots( - attribute_dict["screenshots"].split(" "), urls=True) - processed_images["screenshots"].extend(screenshot_links) - - screenshot_path = os.path.join(activity_location, "screenshots") - if os.path.exists(screenshot_path): - screenshots = glob(os.path.join(screenshot_path, "*.*")) - processed_images["screenshots"].extend(upload_screenshots(screenshots)) - - return processed_images - - -def convert_activity_icon(icon_path): - """Convert activity icon to base64 encoded strign. - - Args: - icon_path: Path where icon is stored - Returns: - Base64 encoded version of SVG Icon - Raises: - BuildProcessErorr: When icon can't be converted or not found - """ - imgur_client = ImgurClient( - app.config['IMGUR_CLIENT_ID'], app.config['IMGUR_CLIENT_SECRET']) - if not (os.path.exists(icon_path) and os.path.isfile(icon_path)): - raise BuildProcessError("Cannot find icon at path %s", icon_path) - - try: - logger.info("Icon path is %s", icon_path) - with open(icon_path, "rb") as f: - img_data = f.read() - encoded_string = base64.b64encode(img_data).decode() - return encoded_string - except Exception as e: - raise BuildProcessError( - "Something unexpected happened while converting the icon. Exception Message %s", e) - - -def get_icon_path(icon_name, activity_name): - """ Returns icon path for an activity. - - Args: - icon_name: Name of the icon - activity_name: Name of the activity - Returns: - Icon path of the activity - Raises: - None - """ - return os.path.join(get_repo_location(activity_name), "activity", icon_name + ".svg") - - -def get_repo_location(name): - return os.path.join(app.config['BUILD_CLONE_REPO'], name) - - -def get_bundle_path(bundle_name): - return os.path.join(app.config['BUILD_BUNDLE_DIR'], bundle_name) - - -def get_parser(activity_file, read_string=False): - parser = configparser.ConfigParser() - if read_string: - try: - parser.read_string(activity_file) - except Exception as e: - raise BuildProcessError( - 'Error parsing metadata file. Error : %s', e) - else: - return parser - else: - if len(parser.read(activity_file)) == 0: - raise BuildProcessError('Error parsing metadata file') - else: - return parser - - -def validate_metadata_attributes(parser, attributes): - MANDATORY_ATTRIBUTES = ['name', 'bundle_id', 'summary', 'license', - 'categories', 'icon', 'activity_version', 'repository', 'activity_version'] - return all(parser.has_option('Activity', attribute) for attribute in attributes) - - -def upload_screenshots(screenshots, urls=False): - """ Uploads screenshots to Imgur. - - Args: - screenshots: List containing screenshots path or urls - urls: Default - False. Determines whether screenshots are paths or urls - Returns: - Returns a list containing coressponding imgur links - Raises: - None - """ - imgur_client = ImgurClient( - app.config['IMGUR_CLIENT_ID'], app.config['IMGUR_CLIENT_SECRET']) - imgur_links = [] - if urls: - for screenshot in screenshots: - result = imgur_client.upload_from_url(screenshot) - imgur_links.append(result['link']) - else: - for screenshot in screenshots: - result = imgur_client.upload_from_path(screenshot) - imgur_links.append(result['link']) - return imgur_links - - -def check_and_download_assets(assets): - - def check_asset(asset): - if asset_name_check(asset['name']): - return asset_manifest_check(asset['browser_download_url'], asset['name']) - return False - - def download_asset(download_url, name): - response = requests.get(download_url, stream=True) - # Save with every block of 1024 bytes - logger.info("Downloading File .. " + name) - with open(os.path.join(app.config['TEMP_BUNDLE_DIR'], name), "wb") as handle: - for block in response.iter_content(chunk_size=1024): - handle.write(block) - return - - def check_info_file(name): - logger.info("Checking For Activity.info") - xo_file = zipfile.ZipFile(os.path.join( - app.config['TEMP_BUNDLE_DIR'], name)) - return any("activity.info" in filename for filename in xo_file.namelist()) - - def asset_name_check(asset_name): - print("Checking for presence of .xo in name of " + asset_name) - return ".xo" in asset_name - - def verify_bundle(bundle_name): - bundle_path = get_bundle_path(bundle_name) - return os.path.exists(bundle_path) and os.path.isfile(bundle_path) - - def asset_manifest_check(download_url, bundle_name): - download_asset(download_url, bundle_name) - if check_info_file(bundle_name): - # Check if that bundle already exists then we don't continue - # Return false if that particular bundle already exists - if verify_bundle(bundle_name): - os.remove(os.path.join( - app.config['TEMP_BUNDLE_DIR'], bundle_name)) - raise BuildProcessError('File %s already exits' % bundle_name) - else: - shutil.move(os.path.join(app.config['TEMP_BUNDLE_DIR'], - bundle_name), app.config['BUILD_BUNDLE_DIR']) - return bundle_name - return False - - for asset in assets: - bundle_name = check_asset(asset) - if bundle_name: - return bundle_name - raise BuildProcessError( - 'No valid bundles were found in this asset release') - - -def clone_repo(url, name, tag): - target_dir = app.config['BUILD_CLONE_REPO'] - if not os.path.isdir(target_dir): - raise BuildProcessError('Directory %s does not exist' % target_dir) - - if os.path.isdir(get_repo_location(name)): - logger.info('Removing existing cloned repo for %s', name) - shutil.rmtree(get_repo_location(name)) - - cmd = ['git', '-c', 'advice.detachedHead=false', '-C', target_dir, - 'clone', '-b', tag, '--depth', '1', url] - - logger.info('Cloning repo %s', url) - if call(cmd) != 0: - raise BuildProcessError('[%s] command has failed' % ' '.join(cmd)) - - -def get_platform_versions(activity, repo_location): - logger.info("Applying heurisitc to determine platform versions") - - def is_web(): - if 'exec' in activity: - return activity['exec'] == 'sugar-activity-web' - return False # Fallback - - GTK3_IMPORT_TYPES = {'sugar3': 3, 'from gi.repository import Gtk': 3, - 'sugar.': 2, 'import pygtk': 2, 'pygtk.require': 2} - - def is_gtk3(): - setup_py_path = os.path.join(repo_location, 'setup.py') - all_files = os.listdir(repo_location) - try_paths = [setup_py_path] + all_files - - for path in try_paths: - if os.path.isfile(path): - with open(path) as f: - text = f.read() - for sign in GTK3_IMPORT_TYPES: - if sign in text: - version = GTK3_IMPORT_TYPES[sign] - return version == 3 - - # Fallback to assuming GTK3 - return True - - OLD_TOOLBAR_SIGNS = ['activity.ActivityToolbox', 'gtk.Toolbar'] - - def has_old_toolbars(): - for path in os.listdir(repo_location): - if os.path.isfile(path): - with open(path) as f: - text = f.read() - for sign in OLD_TOOLBAR_SIGNS: - if sign in text: - return True - return False - - def determine_min_sugar_version(is_gtk3, is_web, has_old_toolbars): - min_sugar_version = '0.100' if is_web else ( - '0.96' if is_gtk3 else ( - '0.86' if not has_old_toolbars else '0.82' - )) - return min_sugar_version - - platform_versions = {} - platform_versions['is_gtk3'] = is_gtk3() - platform_versions['is_web'] = is_web() - platform_versions['has_old_toolbars'] = has_old_toolbars() - platform_versions['min_sugar_version'] = determine_min_sugar_version( - platform_versions['is_gtk3'], platform_versions['is_web'], platform_versions['has_old_toolbars']) - - return platform_versions - - -def get_activity_metadata(name): - def metadata_file_exists(): - repo_dir = get_repo_location(name) - activity_file = os.path.join(repo_dir, "activity/activity.info") - if not os.path.isfile(activity_file): - raise BuildProcessError( - 'Activity file %s does not exist', activity_file - ) - - return activity_file - - def parse_metadata_file(): - parser = get_parser(activity_file) - try: - attributes = dict(parser.items('Activity')) - except configparser.NoSectionError as e: - raise BuildProcessError( - 'Error parsing metadata file. Exception message: %s', e - ) - - return attributes - - activity_file = metadata_file_exists() - return parse_metadata_file() - - -def clean_repo(repo_name): - """ Deletes cloned repository. - - Args: - repo_name: Name of the repository - Returns: - None - Raises: - None - """ - shutil.rmtree(get_repo_location(repo_name)) - - -def invoke_build(name): - def store_bundle(): - dist_dir = os.path.join(get_repo_location(name), 'dist') - if os.path.isdir(dist_dir) and len(os.listdir(dist_dir)) == 1: - bundle_name = os.path.join(dist_dir, os.listdir(dist_dir)[0]) - else: - raise BuildProcessError('Bundle file was not generated correctly') - - try: - shutil.copy2(bundle_name, app.config['BUILD_BUNDLE_DIR']) - stored_bundle = os.path.join( - app.config['BUILD_BUNDLE_DIR'], - os.path.basename(bundle_name) - ) - os.chmod(stored_bundle, 0o644) - except IOError as e: - raise BuildProcessError( - 'Bundle copying has failed: %s', e - ) - - logger.info('Bundle succesfully stored at %s', stored_bundle) - - volume = get_repo_location(name) + ':/activity' - docker_image = app.config['BUILD_DOCKER_IMAGE'] - docker_cmd = ['docker', 'run', '--rm', '-v', volume, docker_image] - logger.info('Running docker command: "%s"', ' '.join(docker_cmd)) - if call(docker_cmd) != 0: - raise BuildProcessError('Docker building process has failed') - - store_bundle() - - -def populate_database(activity, translations, processed_images): - """ Populates MongoDB with MONGO_DBNAME as the target table/collection - - Args: - repo_name: Dictionary containing activity.info attributes - translations: Dictionary containing translations - processed_images: Dictionary processed icon and screenshots - Returns: - None - Raises: - BuildProccessError: When version numbers are not correct (less than the existing one in Database) or when Data cannot be inserted - """ - - def translate_field(field_value, model_class): - """ Helper function which translates a field value and prepares an object with language code as keys - - Args: - field_value: Value of the String to be converted - model_class: Model class whose object needs to prepared for insertion - Returns: - Model class object populated with proper translations of the word if any - Raises: - None - """ - results = [] - for language_code in translations: - if field_value in translations[language_code]: - obj = model_class() - obj[language_code] = field_value - results.append(obj) - return results - - connect(app.config['MONGO_DBNAME']) - try: - metadata = MetaData() - metadata.name.extend(translate_field(activity['name'], Name)) - metadata.bundle_id = activity['bundle_id'] - if 'summary' in activity: - summaries = translate_field(activity['summary'], Summary) - metadata.summary.extend(summaries) - if 'categories' in activity: - metadata.categories = activity['categories'] - metadata.repository = activity['repository'] - metadata.icon = processed_images['icon'] - - release = Release() - release.activity_version = float(activity['activity_version']) - release.release_notes = activity['release_info']['release_notes'] - release.download_url = "https://mock.org/download_url" - release.min_sugar_version = float( - activity['platform_versions']['min_sugar_version']) - release.is_web = activity['platform_versions']['is_web'] - release.has_old_toolbars = activity['platform_versions']['has_old_toolbars'] - release.screenshots.extend(processed_images['screenshots']) - release_time = datetime.datetime.strptime( - activity['release_info']['release_time'], "%Y-%m-%dT%H:%M:%SZ") - release.timestamp = release_time - - release.save() - metadata.add_release(release) - metadata.save() - - except Exception as e: - metadata.delete() - release.delete() - raise BuildProcessError( - "Failed to insert data inside the DB. Error : %s", e) - - -def clean_up(extact_dir): - """ Delete extraction directory of bunlde - Args: - extract: Extraction path of bundle - Returns: - None - Raises: - None - """ - shutil.rmtree(extact_dir) - - -def extract_bundle(bundle_name): - """ Extracts bundle to random directory - Args: - bundle_name: Name of the bundle that needs to be extracted - Returns: - Path where bundle is extracted - Raises: - BuildProcessErorr: If file is corrupted and/or bundle cannot be extracted properly - """ - logger.info("Extracting %s ....", bundle_name) - try: - xo_archive = zipfile.ZipFile(get_bundle_path(bundle_name)) - # Create a random UUID to store the extracted material - random_uuid = uuid.uuid4().hex - # Create the folder with name as random UUID - extract_dir = os.path.join(app.config['TEMP_BUNDLE_DIR'], random_uuid) - os.mkdir(extract_dir) - archive_root_prefix = os.path.commonpath(xo_archive.namelist()) - xo_archive.extractall(path=extract_dir) - extraction_path = os.path.join(extract_dir, archive_root_prefix) - return extraction_path - except Exception as e: - if e.__class__.__name__ is "FileExistsError": - clean_up(extract_dir) - raise BuildProcessError( - "Unable to open archive : %s. Error : %s ", bundle_name, e.__class__) - - -def get_xo_translations(extract_dir): - """ Wrapper function to crawl translations for a bundle - Args: - bundle_name: Directory where bunlde is extracted - Returns: - Dictionary containing translations - Raises: - None - """ - return get_translations(extract_dir) - - -def invoke_asset_build(bundle_name): - def remove_bundle(bundle_name): - logger.info("Removing Bundle : %s", bundle_name) - os.remove(get_bundle_path(bundle_name)) - - def parse_metadata_file(): - parser = get_parser(activity_file, read_string=True) - try: - attributes = dict(parser.items('Activity')) - except configparser.NoSectionError as e: - raise BuildProcessError( - 'Error parsing metadata file. Exception message: %s', e - ) - - return attributes - - def check_bundle(bundle_name): - xo_file = zipfile.ZipFile(get_bundle_path(bundle_name)) - # Find the acitivity_file and return it - for filename in xo_file.namelist(): - if 'activity.info' in filename: - return xo_file.read(filename) - logger.info( - 'Bundle Check has failed. %s is not a valid bundle file ', bundle_name) - raise BuildProcessError( - 'Bundle Check has failed. %s is not a valid bundle file ', bundle_name) - - try: - activity_file = check_bundle(bundle_name) - activity_file = activity_file.decode() - return parse_metadata_file() - except Exception as e: - remove_bundle(bundle_name) - raise BuildProcessError( - 'Error decoding MetaData File. Exception Message: %s', e) diff --git a/aslo/api/exceptions.py b/aslo/api/exceptions.py index b2f6fab..be58eb4 100644 --- a/aslo/api/exceptions.py +++ b/aslo/api/exceptions.py @@ -1,6 +1,10 @@ -class BuildProcessError(Exception): +class ReleaseError(Exception): + pass + + +class BuildProcessError(ReleaseError): pass diff --git a/aslo/api/gh.py b/aslo/api/gh.py new file mode 100644 index 0000000..ce906ff --- /dev/null +++ b/aslo/api/gh.py @@ -0,0 +1,31 @@ +import hmac +import hashlib +from flask import current_app as app +from urllib.parse import urlparse +from github import Github + + +def verify_signature(gh_signature, body, secret): + sha1 = hmac.new(secret.encode(), body, hashlib.sha1).hexdigest() + return hmac.compare_digest('sha1=' + sha1, gh_signature) + + +def auth(): + g = Github(app.config['GITHUB_OAUTH_TOKEN']) + return g + + +def get_developers(repo_url): + o = urlparse(repo_url) + repo = o.path[1:].strip('.git') + + g = auth() + repository = g.get_repo(repo) + contributors = repository.get_contributors() + developers = [] + for c in contributors: + dev = {'email': c.email, 'page': c.html_url, 'avatar': c.avatar_url} + dev['name'] = c.name if c.name else c.login + developers.append(dev) + + return developers diff --git a/aslo/api/i18n.py b/aslo/api/i18n.py new file mode 100644 index 0000000..7616d47 --- /dev/null +++ b/aslo/api/i18n.py @@ -0,0 +1,35 @@ +import os +import glob +import polib +from aslo.celery_app import logger +from .exceptions import ReleaseError + + +def get_language_code(filepath): + basename = os.path.basename(filepath) + return os.path.splitext(basename)[0] + + +def get_translations(repo_path): + logger.info('Getting translations.') + po_files_location = os.path.join(repo_path, 'po/') + + translations = {} + matched_files = glob.glob(os.path.join(po_files_location, '*.po')) + if len(matched_files) == 0: + raise ReleaseError( + "No po files found at location . %s", po_files_location + ) + + po_files = list(map(polib.pofile, matched_files)) + language_codes = list(map(get_language_code, matched_files)) + + # Intialize the dictionary + for language_code in language_codes: + translations[language_code] = {} + + for po_file, language_code in zip(po_files, language_codes): + for entry in po_file.translated_entries(): + translations[language_code][entry.msgid] = entry.msgstr + + return translations diff --git a/aslo/api/release.py b/aslo/api/release.py new file mode 100644 index 0000000..bd2b247 --- /dev/null +++ b/aslo/api/release.py @@ -0,0 +1,351 @@ +import os +import shutil +import configparser +import requests +import zipfile +import datetime +import uuid +import re +import mongoengine as me +import aslo.models.Activity as model +import subprocess as sp +from flask import current_app as app +from aslo.celery_app import logger +from .exceptions import ReleaseError, BuildProcessError +from . import gh +from . import i18n + + +def get_bundle_path(bundle_name): + return os.path.join(app.config['BUILD_BUNDLE_DIR'], bundle_name) + + +def xo_file_exists(assets): + for asset in assets: + if '.xo' in asset['name']: + logger.info('Attached xo file has been found.') + return asset + + logger.info('No attached xo file has been found.') + return None + + +def download_attached_xo(xo): + # save on blocks of 1024 + logger.info("Downloading {} file...".format(xo['name'])) + response = requests.get(xo['browser_download_url'], stream=True) + tmp_bundle_path = os.path.join(app.config['TEMP_BUNDLE_DIR'], xo['name']) + with open(tmp_bundle_path, "wb") as fh: + for block in response.iter_content(chunk_size=1024): + fh.write(block) + + return tmp_bundle_path + + +def verify_and_extract_xo(tmp_bundle_path): + def verify_xo(xo_archive): + logger.info('Searching for activity.info inside xo file.') + valid = any( + 'activity.info' in filename for filename in xo_archive.namelist() + ) + if not valid: + raise ReleaseError('activity.info not found in xo file.') + else: + logger.info('activity.info file has been found in xo file.') + + # TODO: are we going to store this locally and/or in remote server? + bundle_name = os.path.basename(tmp_bundle_path) + bundle_path = get_bundle_path(bundle_name) + if os.path.exists(bundle_path) and os.path.isfile(bundle_path): + raise ReleaseError( + 'Bundle {} already exist.'.format(bundle_name) + ) + + def extract_xo(xo_archive): + random_uuid = uuid.uuid4().hex + extract_dir = os.path.join(app.config['TEMP_BUNDLE_DIR'], random_uuid) + try: + os.mkdir(extract_dir) + except (IOError, FileExistsError) as e: + raise ReleaseError( + 'Failed to created {} directory. Error: {}' + .format(extract_dir, e) + ) + + # Find root_prefix for the xo archive. Usually it's Name.Activity + archive_root_prefix = os.path.commonpath(xo_archive.namelist()) + try: + xo_archive.extractall(path=extract_dir) + extraction_path = os.path.join(extract_dir, archive_root_prefix) + except Exception as e: + logger.exception(e) + + return extraction_path + + xo_archive = zipfile.ZipFile(tmp_bundle_path) + verify_xo(xo_archive) + extraction_path = extract_xo(xo_archive) + + return extraction_path + + +def clone_repo(url, tag, repo_path): + target_dir = app.config['BUILD_CLONE_REPO'] + if not os.path.isdir(target_dir): + raise BuildProcessError('Directory %s does not exist' % target_dir) + + if os.path.isdir(repo_path): + logger.info('Removing existing cloned repo %s', repo_path) + try: + shutil.rmtree(repo_path) + except IOError as e: + raise BuildProcessError( + 'Can\'t remove existing repo {}. Exception: {}' + .format(repo_path, e) + ) + + cmd = ['git', '-c', 'advice.detachedHead=false', '-C', target_dir, + 'clone', '-b', tag, '--depth', '1', url] + + logger.info('Cloning repo %s', url) + if sp.call(cmd) != 0: + raise BuildProcessError('[%s] command has failed' % ' '.join(cmd)) + + +def get_activity_metadata(repo_path): + def metadata_file_exists(): + activity_file = os.path.join(repo_path, 'activity/activity.info') + if not os.path.isfile(activity_file): + raise ReleaseError( + 'Activity file %s does not exist' % activity_file + ) + + return activity_file + + def parse_metadata_file(activity_file): + parser = configparser.ConfigParser() + if len(parser.read(activity_file)) == 0: + raise ReleaseError('Error parsing metadata file') + + try: + attributes = dict(parser.items('Activity')) + except configparser.NoSectionError as e: + raise ReleaseError( + 'Error parsing metadata file. Exception message: %s' % e + ) + + return attributes + + def validate_mandatory_attributes(attributes): + MANDATORY_ATTRIBUTES = ['name', 'bundle_id', 'summary', 'license', + 'categories', 'icon', 'activity_version'] + for attr in MANDATORY_ATTRIBUTES: + if attr not in attributes: + raise ReleaseError( + '%s field missing in activity metadata' % attr + ) + + return True + + logger.info('Getting activity metadata from activity.info file.') + activity_file = metadata_file_exists() + attributes = parse_metadata_file(activity_file) + validate_mandatory_attributes(attributes) + + return attributes + + +def invoke_bundle_build(repo_path): + def check_bundle(): + dist_dir = os.path.join(repo_path, 'dist') + if os.path.isdir(dist_dir) and len(os.listdir(dist_dir)) == 1: + logger.info('Bundle has been built successfully') + return os.path.join(dist_dir, os.listdir(dist_dir)[0]) + else: + raise BuildProcessError('Bundle file was not generated correctly') + + logger.info('Building bundle.') + volume = repo_path + ':/activity' + docker_image = app.config['BUILD_DOCKER_IMAGE'] + docker_cmd = ['docker', 'run', '--rm', '-v', volume, docker_image] + logger.info('Running docker command: "%s"', ' '.join(docker_cmd)) + if sp.call(docker_cmd) != 0: + raise BuildProcessError('Docker building process has failed') + + bundle_path = check_bundle() + return bundle_path + + +def compare_version_in_bundlename_and_metadata(tmp_bundle_path, metadata): + bundle_name = os.path.basename(tmp_bundle_path) + match = re.search('^\w+-(\d+.?\d*).xo$', bundle_name) + bundle_version = match.group(1) if match else None + if metadata['activity_version'] != bundle_version: + raise ReleaseError( + 'Bundle filename version and activity metadata version ' + 'does not match.' + ) + + +def get_sugar_details(activity, repo_path): + logger.info('Applying heuristic to determine min sugar supported version.') + + def is_gtk3(): + GTK3_IMPORT_TYPES = {'sugar3': 3, 'from gi.repository import Gtk': 3, + 'sugar.': 2, 'import pygtk': 2, + 'pygtk.require': 2} + + setup_py_path = os.path.join(repo_path, 'setup.py') + all_files = os.listdir(repo_path) + try_paths = [setup_py_path] + all_files + + for path in try_paths: + if os.path.isfile(path): + with open(path) as f: + text = f.read() + for sign in GTK3_IMPORT_TYPES: + if sign in text: + version = GTK3_IMPORT_TYPES[sign] + return version == 3 + + # Fallback to assuming GTK3 + return True + + def is_web(): + if 'exec' in activity: + return activity['exec'] == 'sugar-activity-web' + return False # Fallback + + def has_old_toolbars(): + OLD_TOOLBAR_SIGNS = ['activity.ActivityToolbox', 'gtk.Toolbar'] + for path in os.listdir(repo_path): + if os.path.isfile(path): + with open(path) as f: + text = f.read() + for sign in OLD_TOOLBAR_SIGNS: + if sign in text: + return True + return False + + def determine_min_sugar_version(is_gtk3, is_web, has_old_toolbars): + min_sugar_version = '0.100' if is_web else ( + '0.96' if is_gtk3 else ( + '0.86' if not has_old_toolbars else '0.82' + )) + return min_sugar_version + + sugar = {} + sugar['is_gtk3'] = is_gtk3() + sugar['is_web'] = is_web() + sugar['has_old_toolbars'] = has_old_toolbars() + sugar['min_sugar_version'] = determine_min_sugar_version( + sugar['is_gtk3'], sugar['is_web'], sugar['has_old_toolbars'] + ) + + return sugar + + +def db_insert_activity(metadata, translations): + def translate_field(field_value, model_class): + obj = model_class() + for language_code in translations: + if field_value in translations[language_code]: + obj[language_code] = translations[language_code][field_value] + return obj + + try: + me.connect(host=app.config['MONGO_URI']) + activity = model.Activity.objects.get(bundle_id=metadata['bundle_id']) + except model.Activity.DoesNotExist: + activity = model.Activity() + activity.bundle_id = metadata['bundle_id'] + + activity.license = metadata['license'] + activity.repository = metadata['repository'] + activity.categories = metadata['categories'].split() + activity.name = translate_field(metadata['name'], model.Name) + activity.summary = translate_field(metadata['summary'], model.Summary) + activity.set_developers(gh.get_developers(metadata['repository'])) + + release = model.Release() + release.activity_version = float(metadata['activity_version']) + release.min_sugar_version = float(metadata['sugar']['min_sugar_version']) + release.is_web = metadata['sugar']['is_web'] + release.has_old_toolbars = metadata['sugar']['has_old_toolbars'] + release.download_url = 'https://mock.org/download_url' + release.release_notes = metadata['release']['notes'] + release.timestamp = datetime.datetime.strptime( + metadata['release']['time'], '%Y-%m-%dT%H:%M:%SZ' + ) + + try: + release.save() + activity.add_release(release) + except me.ValidationError as e: + raise ReleaseError('Failed saving release into db: %s' % e) + else: + try: + logger.info('Saving activity information into db.') + activity.save() + except me.ValidationError as e: + release.delete() + raise ReleaseError('Failed saving activity into db: %s' % e) + + +def store_bundle(tmp_bundle_path): + try: + shutil.copy2(tmp_bundle_path, app.config['BUILD_BUNDLE_DIR']) + stored_bundle = os.path.join( + app.config['BUILD_BUNDLE_DIR'], + os.path.basename(tmp_bundle_path) + ) + os.chmod(stored_bundle, 0o644) + except IOError as e: + raise ReleaseError( + 'Bundle copying has failed: %s', e + ) + + logger.info('Bundle succesfully stored at %s', stored_bundle) + + +def clean_up(tmp_bundle_path, repo_path): + try: + os.remove(tmp_bundle_path) + shutil.rmtree(repo_path) + except IOError as e: + raise ReleaseError('Error removing file: %s' % e) + + +def handle_release(gh_json): + repo_url = gh_json['repository']['clone_url'] + repo_name = gh_json['repository']['name'] + release = gh_json['release'] + tag = release['tag_name'] + xo_asset = None + + if 'assets' in release and len(release['assets']) != 0: + xo_asset = xo_file_exists(release['assets']) + + if xo_asset: + logger.info('[bundle-release] No bundle building process needed.') + tmp_bundle_path = download_attached_xo(xo_asset) + repo_path = verify_and_extract_xo(tmp_bundle_path) + else: + logger.info('[sourcecode-release] Building bundle from source code.') + repo_path = os.path.join(app.config['BUILD_CLONE_REPO'], repo_name) + clone_repo(repo_url, tag, repo_path) + tmp_bundle_path = invoke_bundle_build(repo_path) + + metadata = get_activity_metadata(repo_path) + compare_version_in_bundlename_and_metadata(tmp_bundle_path, metadata) + translations = i18n.get_translations(repo_path) + + metadata['repository'] = repo_url + metadata['release'] = {} + metadata['release']['notes'] = gh_json['release']['body'] + metadata['release']['time'] = gh_json['release']['published_at'] + metadata['sugar'] = get_sugar_details(metadata, repo_path) + + db_insert_activity(metadata, translations) + store_bundle(tmp_bundle_path) + clean_up(tmp_bundle_path, repo_path) diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index 256a65c..ed2716b 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -1,85 +1,14 @@ -from . import build from aslo.celery_app import celery, logger -from .exceptions import BuildProcessError +from .release import handle_release +from .exceptions import ReleaseError, BuildProcessError @celery.task(bind=True) -def build_process(self, gh_json): +def release_process(self, gh_json): try: - release = gh_json['release'] - # TODO: work on cases (bundle attached or not) - # and insert data into DB - # Invoke different build programs depending upon whether it's a source/asset release - if 'assets' in release and len(release['assets']) != 0: - handle_asset_release(gh_json) - else: - handle_source_release(gh_json) + handle_release(gh_json) - except BuildProcessError as e: - logger.exception("Error in activity building process") - return False - - logger.info('Activity building process finished !') - - -def handle_source_release(gh_json): - logger.info('Building for a source release') - try: - url = gh_json['repository']['clone_url'] - name = gh_json['repository']['name'] - release = gh_json['release'] - tag = release['tag_name'] - - build.clone_repo(url, name, tag) - activity = build.get_activity_metadata(name) - # Insert/Override repository tag - activity['repository'] = url - repo_location = build.get_repo_location(name) - build.invoke_build(name) - translations = build.get_translations(repo_location) - processed_images = build.process_image_assets(activity, repo_location) - activity['platform_versions'] = build.get_platform_versions( - activity, repo_location) - activity['release_info'] = {} - activity['release_info']['release_notes'] = gh_json['release']['body'] - activity['release_info']['release_time'] = gh_json['release']['published_at'] - build.clean_repo(name) - build.populate_database(activity, translations, processed_images) - logger.info(activity) - logger.info(processed_images) - logger.info(translations["es"]) - except BuildProcessError as e: - logger.exception("Error in activity building process") - return False - - -def handle_asset_release(gh_json): - logger.info('Building for a asset release') - try: - url = gh_json['repository']['clone_url'] - name = gh_json['repository']['name'] - release = gh_json['release'] - tag = release['tag_name'] - - bundle_name = build.check_and_download_assets(release['assets']) - activity = build.invoke_asset_build(bundle_name) - # Insert/Override repository tag - activity['repository'] = url - extracted_bundle = build.extract_bundle(bundle_name) - processed_images = build.process_image_assets( - activity, extracted_bundle) - activity['platform_versions'] = build.get_platform_versions( - activity, extracted_bundle) - activity['release_info'] = {} - activity['release_info']['release_notes'] = gh_json['release']['body'] - activity['release_info']['release_time'] = gh_json['release']['published_at'] - - translations = build.get_xo_translations(extracted_bundle) - build.populate_database(activity, translations, processed_images) - logger.info(processed_images) - logger.info(translations["es"]) - logger.info(activity) - - except BuildProcessError as e: - logger.exception("Error in activity building process") - return False + except (ReleaseError, BuildProcessError): + logger.exception("Error in activity release process!") + else: + logger.info('Activity release process has finished successfully!') diff --git a/aslo/api/views.py b/aslo/api/views.py index 2ff250e..119dab6 100644 --- a/aslo/api/views.py +++ b/aslo/api/views.py @@ -1,8 +1,8 @@ from flask import request, current_app as app from . import api from .exceptions import ApiHttpError -from .tasks import build_process -from .utils import verify_signature +from .tasks import release_process +from .gh import verify_signature @api.route('/hook', methods=['POST']) @@ -28,6 +28,6 @@ def hook(): if not valid_signature: raise ApiHttpError('Invalid Signature', 400) - build_process.apply_async(args=[body_json]) + release_process.apply_async(args=[body_json]) return "{'status_code': 200, 'message': 'OK'}", 200 diff --git a/aslo/default_settings.py b/aslo/default_settings.py index 8025546..de58479 100644 --- a/aslo/default_settings.py +++ b/aslo/default_settings.py @@ -22,6 +22,8 @@ def env(variable, fallback_value=None): # GITHUB WEBHOOK GITHUB_HOOK_SECRET = env('GITHUB_HOOK_SECRET', '') +# GITHUB OAUTH ACCESS TOKEN +GITHUB_OAUTH_TOKEN = env('GITHUB_OAUTH_TOKEN', '') # CELERY REDIS_URI = env('REDIS_URL', 'redis://localhost:6379/1') diff --git a/aslo/models/Activity.py b/aslo/models/Activity.py index 7d93706..7d2aa39 100644 --- a/aslo/models/Activity.py +++ b/aslo/models/Activity.py @@ -1,59 +1,72 @@ -from mongoengine import Document, StringField, ListField, DynamicEmbeddedDocument, EmbeddedDocumentListField, IntField, FloatField, URLField, BooleanField, DateTimeField, ReferenceField, ValidationError +import mongoengine as me -class Name(DynamicEmbeddedDocument): +class Name(me.DynamicEmbeddedDocument): # Empty class to avoid any empty object since we will have dynamic fields def __str__(self): return self.to_json() + def to_dict(self): + return self.to_mongo.to_dict() -class Summary(DynamicEmbeddedDocument): + +class Summary(me.DynamicEmbeddedDocument): # Empty class to avoid any empty object since we will have dynamic fields def __str__(self): return self.to_json() + def to_dict(self): + return self.to_mongo.to_dict() + -class Developer(DynamicEmbeddedDocument): - name = StringField(required=True) - email = StringField() - page = StringField() +class Developer(me.DynamicEmbeddedDocument): + name = me.StringField(required=True) + email = me.EmailField() + page = me.URLField() + avatar = me.URLField() def __str__(self): return self.to_json() + def to_dict(self): + return self.to_mongo.to_dict() + -class Release(Document): - # We can aslo use StringField for storing versions - activity_version = FloatField(required=True) - release_notes = StringField(required=True) - # Use custom class bound method to calculate min_sugar_version ? - min_sugar_version = FloatField(required=True) +class Release(me.Document): + activity_version = me.FloatField(required=True) + release_notes = me.StringField(required=True) + min_sugar_version = me.FloatField(required=True) # Also known as xo_url - download_url = URLField(required=True) - is_web = BooleanField(required=True, default=False) - is_gtk = BooleanField(required=True, default=False) - has_old_toolbars = BooleanField(required=False, default=False) + download_url = me.URLField(required=True) + is_web = me.BooleanField(required=True, default=False) + is_gtk3 = me.BooleanField(required=True, default=False) + has_old_toolbars = me.BooleanField(required=False, default=False) # Timestamp when the release was made - screenshots = ListField(URLField(required=False), required=False) - timestamp = DateTimeField(required=True) + timestamp = me.DateTimeField(required=True) + + def __str__(self): + return self.to_json() + + def to_dict(self): + return self.to_mongo.to_dict() + + +class Activity(me.Document): + bundle_id = me.StringField(required=True) + name = me.EmbeddedDocumentField(Name, required=True) + summary = me.EmbeddedDocumentField(Summary, required=True) + categories = me.ListField(me.StringField(max_length=30)) + repository = me.StringField(required=True) + license = me.StringField(required=True) + developers = me.EmbeddedDocumentListField(Developer, required=True) + latest_release = me.ReferenceField(Release) + previous_releases = me.ListField(me.ReferenceField(Release)) def __str__(self): - self.to_json() - - -class MetaData(Document): - name = EmbeddedDocumentListField(Name, required=True) - bundle_id = StringField(required=True) - summary = EmbeddedDocumentListField(Summary, required=False, default=[]) - # We aslo use ListField for categories but then we need to join and slice it from the MetaData - categories = StringField(default="") - #activity_version = StringField(required=True) - repository = StringField(required=True) - developers = EmbeddedDocumentListField(Developer, required=False) - # Base64 encoded string to store images - icon = StringField(required=True) - latest_release = ReferenceField(Release) - previous_releases = ListField(ReferenceField(Release)) + return self.to_json() + + def to_dict(self): + return self.to_mongo.to_dict() def add_release(self, release): # first release @@ -63,14 +76,24 @@ def add_release(self, release): if self.latest_release.activity_version >= release.activity_version: release.delete() - raise ValidationError( + raise me.ValidationError( 'New activity release version {} is less or equal than the ' - 'current version {}' - .format( + 'current version {}'.format( release.activity_version, self.latest_release.activity_version - ) + ) ) self.previous_releases.append(self.latest_release) self.latest_release = release + + def set_developers(self, developers): + devs = [] + for developer in developers: + dev = Developer() + dev.name = developer['name'] + dev.email = developer['email'] + dev.page = developer['page'] + dev.avatar = developer['avatar'] + devs.append(dev) + self.developers = devs diff --git a/requirements.txt b/requirements.txt index dbe53b7..34ca2b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ flask_bootstrap celery redis pymongo -requests mongoengine +requests polib -imgurpython \ No newline at end of file +pygithub From a429253b14b48fa93f52f0ba3f99d3e1da147185 Mon Sep 17 00:00:00 2001 From: Samuel Cantero Date: Sun, 16 Jul 2017 21:15:02 -0400 Subject: [PATCH 29/57] Add service layer and image management --- aslo/__init__.py | 6 +- aslo/api/exceptions.py | 4 + aslo/api/i18n.py | 8 ++ aslo/api/img.py | 73 +++++++++++++++++ aslo/api/release.py | 84 +++++++------------ aslo/api/utils.py | 7 -- aslo/api/views.py | 1 + aslo/celery_app.py | 3 - aslo/models/Activity.py | 99 ----------------------- aslo/models/__init__.py | 22 +++++ aslo/models/activity.py | 30 +++++++ aslo/models/base.py | 34 ++++++++ aslo/models/release.py | 22 +++++ aslo/persistence/__init__.py | 0 aslo/persistence/access.py | 25 ++++++ aslo/persistence/activity.py | 20 +++++ aslo/persistence/release.py | 10 +++ aslo/service/__init__.py | 5 ++ aslo/service/activity.py | 90 +++++++++++++++++++++ aslo/{default_settings.py => settings.py} | 0 20 files changed, 379 insertions(+), 164 deletions(-) create mode 100644 aslo/api/img.py delete mode 100644 aslo/api/utils.py delete mode 100644 aslo/models/Activity.py create mode 100644 aslo/models/__init__.py create mode 100644 aslo/models/activity.py create mode 100644 aslo/models/base.py create mode 100644 aslo/models/release.py create mode 100644 aslo/persistence/__init__.py create mode 100644 aslo/persistence/access.py create mode 100644 aslo/persistence/activity.py create mode 100644 aslo/persistence/release.py create mode 100644 aslo/service/__init__.py create mode 100644 aslo/service/activity.py rename aslo/{default_settings.py => settings.py} (100%) diff --git a/aslo/__init__.py b/aslo/__init__.py index 760ac13..c6942d8 100644 --- a/aslo/__init__.py +++ b/aslo/__init__.py @@ -7,7 +7,7 @@ def init_app(): app = Flask(__name__) - app.config.from_object('aslo.default_settings') + app.config.from_object('aslo.settings') # init bootstrap Bootstrap(app) @@ -33,4 +33,8 @@ def init_app(): if app.config['DEBUG']: logger.setLevel(logging.DEBUG) + # setup db + from .service import setup_db + setup_db(app) + return app diff --git a/aslo/api/exceptions.py b/aslo/api/exceptions.py index be58eb4..5813a4d 100644 --- a/aslo/api/exceptions.py +++ b/aslo/api/exceptions.py @@ -8,6 +8,10 @@ class BuildProcessError(ReleaseError): pass +class ScreenshotDoesNotExist(ReleaseError): + pass + + class ApiHttpError(Exception): def __init__(self, message, status_code=None): diff --git a/aslo/api/i18n.py b/aslo/api/i18n.py index 7616d47..9aaaef6 100644 --- a/aslo/api/i18n.py +++ b/aslo/api/i18n.py @@ -33,3 +33,11 @@ def get_translations(repo_path): translations[language_code][entry.msgid] = entry.msgstr return translations + + +def translate_field(field_value, translations): + d = {} + for language_code in translations: + if field_value in translations[language_code]: + d[language_code] = translations[language_code][field_value] + return d diff --git a/aslo/api/img.py b/aslo/api/img.py new file mode 100644 index 0000000..3a84229 --- /dev/null +++ b/aslo/api/img.py @@ -0,0 +1,73 @@ +import os +import glob +import hashlib + +from flask import current_app as app +from aslo.service import activity as activity_service +from .exceptions import ReleaseError, ScreenshotDoesNotExist +from imgurpython import ImgurClient + + +def get_icon(repo_path, icon_name): + icon_path = os.path.join(repo_path, 'activity', icon_name + '.svg') + if not os.path.isfile(icon_path): + raise ReleaseError('Icon file %s is missing' % icon_path) + + try: + with open(icon_path, 'rb') as f: + return f.read() + except IOError as e: + raise ReleaseError( + 'Failed opening icon file %s with error: %s' % (icon_path, e) + ) + + +def get_img_hash(img_path, blocksize=2**20): + h = hashlib.sha1() + with open(img_path, 'rb') as f: + for chunk in iter(lambda: f.read(blocksize), b''): + h.update(chunk) + + return str(h.hexdigest()) + + +def upload_img_to_imgur(image_path): + imgur_client = ImgurClient( + app.config['IMGUR_CLIENT_ID'], app.config['IMGUR_CLIENT_SECRET'] + ) + result = imgur_client.upload_from_path(image_path) + return (result['link'], result['deletehash']) + + +def get_screenshots(repo_path, bundle_id): + screenshots_path = os.path.join(repo_path, 'screenshots') + if not os.path.isdir(screenshots_path): + raise ScreenshotDoesNotExist( + 'Screenshots folder %s doesn\'t exit' % screenshots_path + ) + + old_screenshots = activity_service.get_all_screenshots(bundle_id) + new_screenshots = {} + for lang in os.listdir(screenshots_path): + if not os.path.isdir(os.path.join(screenshots_path, lang)): + continue + + new_screenshots[lang] = {} + images = os.path.join(screenshots_path, lang, '*.*') + for image in glob.glob(images): + if not image.endswith('.png'): + continue + + _hash = get_img_hash(image) + new_screenshots[lang][_hash] = {} + if lang in old_screenshots and _hash in old_screenshots[lang]: + new_screenshots[lang][_hash] = old_screenshots[lang][_hash] + continue + + link, deletehash = upload_img_to_imgur(image) + new_screenshots[lang][_hash] = (link, deletehash) + + if len(new_screenshots[lang]) == 0: + del new_screenshots[lang] + + return new_screenshots diff --git a/aslo/api/release.py b/aslo/api/release.py index bd2b247..28a8a75 100644 --- a/aslo/api/release.py +++ b/aslo/api/release.py @@ -2,18 +2,19 @@ import shutil import configparser import requests -import zipfile import datetime +import zipfile import uuid import re -import mongoengine as me -import aslo.models.Activity as model import subprocess as sp + from flask import current_app as app +from aslo.service import activity as activity_service from aslo.celery_app import logger -from .exceptions import ReleaseError, BuildProcessError +from .exceptions import ReleaseError, BuildProcessError, ScreenshotDoesNotExist from . import gh from . import i18n +from . import img def get_bundle_path(bundle_name): @@ -245,53 +246,6 @@ def determine_min_sugar_version(is_gtk3, is_web, has_old_toolbars): return sugar -def db_insert_activity(metadata, translations): - def translate_field(field_value, model_class): - obj = model_class() - for language_code in translations: - if field_value in translations[language_code]: - obj[language_code] = translations[language_code][field_value] - return obj - - try: - me.connect(host=app.config['MONGO_URI']) - activity = model.Activity.objects.get(bundle_id=metadata['bundle_id']) - except model.Activity.DoesNotExist: - activity = model.Activity() - activity.bundle_id = metadata['bundle_id'] - - activity.license = metadata['license'] - activity.repository = metadata['repository'] - activity.categories = metadata['categories'].split() - activity.name = translate_field(metadata['name'], model.Name) - activity.summary = translate_field(metadata['summary'], model.Summary) - activity.set_developers(gh.get_developers(metadata['repository'])) - - release = model.Release() - release.activity_version = float(metadata['activity_version']) - release.min_sugar_version = float(metadata['sugar']['min_sugar_version']) - release.is_web = metadata['sugar']['is_web'] - release.has_old_toolbars = metadata['sugar']['has_old_toolbars'] - release.download_url = 'https://mock.org/download_url' - release.release_notes = metadata['release']['notes'] - release.timestamp = datetime.datetime.strptime( - metadata['release']['time'], '%Y-%m-%dT%H:%M:%SZ' - ) - - try: - release.save() - activity.add_release(release) - except me.ValidationError as e: - raise ReleaseError('Failed saving release into db: %s' % e) - else: - try: - logger.info('Saving activity information into db.') - activity.save() - except me.ValidationError as e: - release.delete() - raise ReleaseError('Failed saving activity into db: %s' % e) - - def store_bundle(tmp_bundle_path): try: shutil.copy2(tmp_bundle_path, app.config['BUILD_BUNDLE_DIR']) @@ -338,14 +292,36 @@ def handle_release(gh_json): metadata = get_activity_metadata(repo_path) compare_version_in_bundlename_and_metadata(tmp_bundle_path, metadata) + translations = i18n.get_translations(repo_path) + metadata['i18n_name'] = i18n.translate_field(metadata['name'], + translations) + metadata['i18n_summary'] = i18n.translate_field(metadata['summary'], + translations) metadata['repository'] = repo_url + metadata['developers'] = gh.get_developers(metadata['repository']) + metadata['icon_bin'] = img.get_icon(repo_path, metadata['icon']) + + try: + screenshots = img.get_screenshots(repo_path, metadata['bundle_id']) + except ScreenshotDoesNotExist as e: + screenshots = {} + logger.info(e) + finally: + metadata['screenshots'] = screenshots + + metadata['sugar'] = get_sugar_details(metadata, repo_path) + metadata['release'] = {} metadata['release']['notes'] = gh_json['release']['body'] - metadata['release']['time'] = gh_json['release']['published_at'] - metadata['sugar'] = get_sugar_details(metadata, repo_path) + metadata['release']['time'] = datetime.datetime.strptime( + gh_json['release']['published_at'], '%Y-%m-%dT%H:%M:%SZ' + ) - db_insert_activity(metadata, translations) + logger.info('Inserting activity into db.') + activity_service.insert_activity(metadata) + logger.info('Saving bundle.') store_bundle(tmp_bundle_path) + logger.info('Cleaning up.') clean_up(tmp_bundle_path, repo_path) diff --git a/aslo/api/utils.py b/aslo/api/utils.py deleted file mode 100644 index 7cf9da2..0000000 --- a/aslo/api/utils.py +++ /dev/null @@ -1,7 +0,0 @@ -import hmac -import hashlib - - -def verify_signature(gh_signature, body, secret): - sha1 = hmac.new(secret.encode(), body, hashlib.sha1).hexdigest() - return hmac.compare_digest('sha1=' + sha1, gh_signature) diff --git a/aslo/api/views.py b/aslo/api/views.py index 119dab6..d65ce51 100644 --- a/aslo/api/views.py +++ b/aslo/api/views.py @@ -1,4 +1,5 @@ from flask import request, current_app as app + from . import api from .exceptions import ApiHttpError from .tasks import release_process diff --git a/aslo/celery_app.py b/aslo/celery_app.py index 0cb4691..da94c76 100644 --- a/aslo/celery_app.py +++ b/aslo/celery_app.py @@ -15,9 +15,6 @@ def __call__(self, *args, **kwargs): with app.app_context(): return TaskBase.__call__(self, *args, **kwargs) -# def on_failure(self, exc, task_id, args, kwargs, einfo): -# print('{0!r} failed: {1!r}'.format(task_id, exc)) - celery.Task = ContextTask celery.config_from_object(app.config, namespace='CELERY') app.celery = celery diff --git a/aslo/models/Activity.py b/aslo/models/Activity.py deleted file mode 100644 index 7d2aa39..0000000 --- a/aslo/models/Activity.py +++ /dev/null @@ -1,99 +0,0 @@ -import mongoengine as me - - -class Name(me.DynamicEmbeddedDocument): - # Empty class to avoid any empty object since we will have dynamic fields - def __str__(self): - return self.to_json() - - def to_dict(self): - return self.to_mongo.to_dict() - - -class Summary(me.DynamicEmbeddedDocument): - # Empty class to avoid any empty object since we will have dynamic fields - def __str__(self): - return self.to_json() - - def to_dict(self): - return self.to_mongo.to_dict() - - -class Developer(me.DynamicEmbeddedDocument): - name = me.StringField(required=True) - email = me.EmailField() - page = me.URLField() - avatar = me.URLField() - - def __str__(self): - return self.to_json() - - def to_dict(self): - return self.to_mongo.to_dict() - - -class Release(me.Document): - activity_version = me.FloatField(required=True) - release_notes = me.StringField(required=True) - min_sugar_version = me.FloatField(required=True) - # Also known as xo_url - download_url = me.URLField(required=True) - is_web = me.BooleanField(required=True, default=False) - is_gtk3 = me.BooleanField(required=True, default=False) - has_old_toolbars = me.BooleanField(required=False, default=False) - # Timestamp when the release was made - timestamp = me.DateTimeField(required=True) - - def __str__(self): - return self.to_json() - - def to_dict(self): - return self.to_mongo.to_dict() - - -class Activity(me.Document): - bundle_id = me.StringField(required=True) - name = me.EmbeddedDocumentField(Name, required=True) - summary = me.EmbeddedDocumentField(Summary, required=True) - categories = me.ListField(me.StringField(max_length=30)) - repository = me.StringField(required=True) - license = me.StringField(required=True) - developers = me.EmbeddedDocumentListField(Developer, required=True) - latest_release = me.ReferenceField(Release) - previous_releases = me.ListField(me.ReferenceField(Release)) - - def __str__(self): - return self.to_json() - - def to_dict(self): - return self.to_mongo.to_dict() - - def add_release(self, release): - # first release - if not self.latest_release and len(self.previous_releases) == 0: - self.latest_release = release - return - - if self.latest_release.activity_version >= release.activity_version: - release.delete() - raise me.ValidationError( - 'New activity release version {} is less or equal than the ' - 'current version {}'.format( - release.activity_version, - self.latest_release.activity_version - ) - ) - - self.previous_releases.append(self.latest_release) - self.latest_release = release - - def set_developers(self, developers): - devs = [] - for developer in developers: - dev = Developer() - dev.name = developer['name'] - dev.email = developer['email'] - dev.page = developer['page'] - dev.avatar = developer['avatar'] - devs.append(dev) - self.developers = devs diff --git a/aslo/models/__init__.py b/aslo/models/__init__.py new file mode 100644 index 0000000..388f127 --- /dev/null +++ b/aslo/models/__init__.py @@ -0,0 +1,22 @@ + + +class MongoDBAccess(): + def __init__(self, model): + self._model = model + + def get_by_id(self, value): + for model_object in self._model.objects(id=value): + return model_object + raise ValueError('{} with id "{}" does not exist.'.format( + self._model.__name__, value)) + + def get_all(self): + return self._model.objects() + + @staticmethod + def add_or_update(model_object): + return model_object.save() + + @staticmethod + def delete(model_object): + model_object.delete() diff --git a/aslo/models/activity.py b/aslo/models/activity.py new file mode 100644 index 0000000..533f0db --- /dev/null +++ b/aslo/models/activity.py @@ -0,0 +1,30 @@ +import mongoengine as me + +from . import MongoDBAccess +from .base import AsloBaseModel +from .release import ReleaseModel + + +class DeveloperModel(me.DynamicEmbeddedDocument): + name = me.StringField(required=True) + email = me.EmailField() + page = me.URLField() + avatar = me.URLField() + + +class ActivityModel(AsloBaseModel): + meta = {'collection': 'activity'} + bundle_id = me.StringField(required=True, unique=True) + name = me.DictField(required=True) + summary = me.DictField(required=True) + categories = me.ListField(me.StringField(max_length=30)) + repository = me.StringField(required=True) + license = me.StringField(required=True) + icon = me.BinaryField(required=False, max_bytes=1000000) + icon_hash = me.StringField(required=False) + developers = me.EmbeddedDocumentListField(DeveloperModel, required=True) + latest_release = me.ReferenceField(ReleaseModel) + previous_releases = me.ListField(me.ReferenceField(ReleaseModel)) + + +activity_access = MongoDBAccess(ActivityModel) diff --git a/aslo/models/base.py b/aslo/models/base.py new file mode 100644 index 0000000..d46bc6f --- /dev/null +++ b/aslo/models/base.py @@ -0,0 +1,34 @@ +import mongoengine as me + + +class AsloBaseModel(me.Document): + + # http://docs.mongoengine.org/guide/defining-documents.html#abstract-classes + meta = { + 'abstract': True + } + + def to_dict(self): + return self.to_mongo.to_dict() + + +class MongoDBAccess(): + def __init__(self, model): + self._model = model + + def get_by_id(self, value): + for model_object in self._model.objects(id=value): + return model_object + raise ValueError('{} with id "{}" does not exist.'.format( + self._model.__name__, value)) + + def get_all(self): + return self._model.objects() + + @staticmethod + def add_or_update(model_object): + return model_object.save() + + @staticmethod + def delete(model_object): + model_object.delete() diff --git a/aslo/models/release.py b/aslo/models/release.py new file mode 100644 index 0000000..9bef992 --- /dev/null +++ b/aslo/models/release.py @@ -0,0 +1,22 @@ +import mongoengine as me + +from . import MongoDBAccess +from .base import AsloBaseModel + + +class ReleaseModel(AsloBaseModel): + meta = {'collection': 'release'} + activity_version = me.FloatField(required=True) + release_notes = me.StringField(required=True) + min_sugar_version = me.FloatField(required=True) + # Also known as xo_url + download_url = me.URLField(required=True) + is_web = me.BooleanField(required=True, default=False) + is_gtk3 = me.BooleanField(required=True, default=False) + has_old_toolbars = me.BooleanField(required=False, default=False) + screenshots = me.DictField() + # Timestamp when the release was made + timestamp = me.DateTimeField(required=True) + + +release_access = MongoDBAccess(ReleaseModel) diff --git a/aslo/persistence/__init__.py b/aslo/persistence/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aslo/persistence/access.py b/aslo/persistence/access.py new file mode 100644 index 0000000..70dd666 --- /dev/null +++ b/aslo/persistence/access.py @@ -0,0 +1,25 @@ +import abc + + +class Access(metaclass=abc.ABCMeta): + + @classmethod + @abc.abstractmethod + def _get_impl(cls): + pass + + @classmethod + def get_by_id(cls, _id): + cls._get_impl().get_by_id(_id) + + @classmethod + def get_all(cls): + cls._get_impl().get_all() + + @classmethod + def add_or_update(cls, obj): + cls._get_impl().add_or_update(obj) + + @classmethod + def delete(cls, obj): + cls._get_impl().delete(obj) diff --git a/aslo/persistence/activity.py b/aslo/persistence/activity.py new file mode 100644 index 0000000..e54b412 --- /dev/null +++ b/aslo/persistence/activity.py @@ -0,0 +1,20 @@ +from .access import Access +from aslo.models.activity import activity_access + + +class Activity(Access): + impl = activity_access + + @classmethod + def _get_impl(cls): + return cls.impl + + @classmethod + def get_by_bundle_id(cls, value): + _model = cls._get_impl()._model + try: + activity = _model.objects.get(bundle_id=value) + except _model.DoesNotExist: + activity = None + + return activity diff --git a/aslo/persistence/release.py b/aslo/persistence/release.py new file mode 100644 index 0000000..4fc3c48 --- /dev/null +++ b/aslo/persistence/release.py @@ -0,0 +1,10 @@ +from .access import Access +from aslo.models.release import release_access + + +class Release(Access): + impl = release_access + + @classmethod + def _get_impl(cls): + return cls.impl diff --git a/aslo/service/__init__.py b/aslo/service/__init__.py new file mode 100644 index 0000000..e07ed60 --- /dev/null +++ b/aslo/service/__init__.py @@ -0,0 +1,5 @@ +import mongoengine as me + + +def setup_db(app): + me.connect(host=app.config['MONGO_URI'], connect=False) diff --git a/aslo/service/activity.py b/aslo/service/activity.py new file mode 100644 index 0000000..a134699 --- /dev/null +++ b/aslo/service/activity.py @@ -0,0 +1,90 @@ +import mongoengine as me +import hashlib + +from aslo.api.exceptions import ReleaseError +from aslo.models.activity import ActivityModel, DeveloperModel +from aslo.models.release import ReleaseModel +from aslo.persistence.release import Release +from aslo.persistence.activity import Activity + + +def add_release(activity, release): + # first release + if not activity.latest_release and len(activity.previous_releases) == 0: + activity.latest_release = release + return + + if activity.latest_release.activity_version >= release.activity_version: + Release.delete(release) + raise me.ValidationError( + 'New activity release version {} is less or equal than the ' + 'current version {}'.format( + release.activity_version, + activity.latest_release.activity_version + ) + ) + + activity.previous_releases.append(activity.latest_release) + activity.latest_release = release + + +def set_developers(activity, developers): + devs = [] + for developer in developers: + dev = DeveloperModel() + dev.name = developer['name'] + dev.email = developer['email'] + dev.page = developer['page'] + dev.avatar = developer['avatar'] + devs.append(dev) + activity.developers = devs + + +def get_all_screenshots(bundle_id): + screenshots = {} + activity = Activity.get_by_bundle_id(bundle_id) + if activity: + release = activity.latest_release + screenshots = release.screenshots + + return screenshots + + +def insert_activity(data): + activity = Activity.get_by_bundle_id(data['bundle_id']) + if activity is None: + activity = ActivityModel() + activity.bundle_id = data['bundle_id'] + + activity.license = data['license'] + activity.repository = data['repository'] + activity.categories = data['categories'].split() + activity.name = data['i18n_name'] + activity.summary = data['i18n_summary'] + set_developers(activity, data['developers']) + icon_hash = hashlib.sha1(data['icon_bin']).hexdigest() + if icon_hash != activity.icon_hash: + activity.icon = data['icon_bin'] + activity.icon_hash = icon_hash + + release = ReleaseModel() + release.activity_version = float(data['activity_version']) + release.min_sugar_version = float(data['sugar']['min_sugar_version']) + release.is_web = data['sugar']['is_web'] + release.has_old_toolbars = data['sugar']['has_old_toolbars'] + release.download_url = 'https://mock.org/download_url' + release.release_notes = data['release']['notes'] + release.timestamp = data['release']['time'] + release.screenshots = data['screenshots'] + + try: + Release.add_or_update(release) + add_release(activity, release) + except me.ValidationError as e: + raise ReleaseError('Failed saving release into db: %s' % e) + else: + try: + Activity.add_or_update(activity) + except me.ValidationError as e: + Release.delete(release) + raise ReleaseError('Failed saving activity into db: %s' % e) diff --git a/aslo/default_settings.py b/aslo/settings.py similarity index 100% rename from aslo/default_settings.py rename to aslo/settings.py From d745799e623976674cb66cd303135a26abf94961 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Wed, 12 Jul 2017 19:55:49 +0530 Subject: [PATCH 30/57] Encrypt and add Github token --- .env.enc | Bin 128 -> 192 bytes .travis.yml | 6 ++---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.env.enc b/.env.enc index a6fdcd889fb367220c60d188683744acd960938b..864f2405f56ec82bef3e48c724a006b0a6a2ac4a 100644 GIT binary patch literal 192 zcmV;x06+iNPq2=Od*W84U~c*Bx2c;LnYs2pvq>Q+AkiZNm>TY22WijA3|>I{87^XJ z>N;8vW)=2KR5{hPcwLQhZlh>#=o*gwxPuF$Awlnh)_-WMoZV!X^}CwY$7g!Qi?Dw- zf&lkdVD2=JeA%bs)O>CI9g7m0ASMq literal 128 zcmV-`0Du2N;{QI_2rKcb{a_9gJif1y)sSYgm4G#@d*=%9XMkEo&T2UpWT}9E^$AG< z2~3xwNY_6*SG-e%N^{_Jydp6DrAJYtM$$imqNLTP=89u*dl#Y} ibbzcN4@t`V4D;4$s2aY~h#Tc+h-V%pXRb!7EKabXxjlFQ diff --git a/.travis.yml b/.travis.yml index 9017216..8722a98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,5 @@ before_install: - openssl aes-256-cbc -K $encrypted_ad8f1517fc45_key -iv $encrypted_ad8f1517fc45_iv -in .env.enc -out .env -d install: pip install -r requirements.txt -# Run server only for 50 seconds, some time given for Imgur Client to verify keys, ideally it takes less than 2 seconds to validate and boot up whole server -# Timeout returns 124 if command timeouts without any problems -script: timeout -sHUP 0.5m bash start.sh; if [ "$?" -eq 124 ]; then exit 0 ; else exit 1 ; fi - +script: timeout -sHUP 0.5m bash start.sh; if [ "$?" -eq 124 ]; then exit 0 ; else + exit 1 ; fi From 7a0f4969e595e569809c8f6ca4832c37c4aa0303 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Thu, 13 Jul 2017 15:34:31 +0530 Subject: [PATCH 31/57] Add the Bot communication. Bot will post and pre build with the build status as comments on the release tag commit. Removed ununsed aslo/api/utils.py --- aslo/api/gh.py | 16 ++++++++++++++++ aslo/api/release.py | 4 +++- aslo/api/tasks.py | 12 ++++++++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/aslo/api/gh.py b/aslo/api/gh.py index ce906ff..de839eb 100644 --- a/aslo/api/gh.py +++ b/aslo/api/gh.py @@ -29,3 +29,19 @@ def get_developers(repo_url): developers.append(dev) return developers + + +def find_tag_commit(repo_name, tag_name): + g = auth() + tags = g.get_repo(repo_name).get_tags() + tag_commit = None + for tag in tags: + if tag.name == tag_name: + tag_commit = tag.commit + + return tag_commit + + +# No needed though :| (just a wrapper) +def comment_on_commit(commit, message): + commit.create_comment(message) diff --git a/aslo/api/release.py b/aslo/api/release.py index 28a8a75..3ed23aa 100644 --- a/aslo/api/release.py +++ b/aslo/api/release.py @@ -276,7 +276,9 @@ def handle_release(gh_json): release = gh_json['release'] tag = release['tag_name'] xo_asset = None - + tag_commit = gh.find_tag_commit(gh_json['repository']['full_name'],tag) + #TODO: Extract message to constants file + gh.comment_on_commit(tag_commit,"Build has started :hourglass_flowing_sand:") if 'assets' in release and len(release['assets']) != 0: xo_asset = xo_file_exists(release['assets']) diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index ed2716b..a873086 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -1,14 +1,22 @@ from aslo.celery_app import celery, logger from .release import handle_release from .exceptions import ReleaseError, BuildProcessError +from .gh import find_tag_commit, comment_on_commit @celery.task(bind=True) def release_process(self, gh_json): try: handle_release(gh_json) - - except (ReleaseError, BuildProcessError): + except (ReleaseError, BuildProcessError) as error: logger.exception("Error in activity release process!") + tag_commit = find_tag_commit( + gh_json['repository']['full_name'], gh_json['release']['tag_name']) + comment_on_commit( + tag_commit, "Build Failed :x: Details: {}".format(error)) else: logger.info('Activity release process has finished successfully!') + tag_commit = find_tag_commit( + gh_json['repository']['full_name'], gh_json['release']['tag_name']) + comment_on_commit( + tag_commit, "Build Passed :white_check_mark: Download url will be provided soon :link:") From f118e304d60082565569b5803b91e1ac522cba9a Mon Sep 17 00:00:00 2001 From: Samuel Cantero Date: Sun, 16 Jul 2017 21:54:34 -0400 Subject: [PATCH 32/57] pep8 fixes --- aslo/api/gh.py | 1 - aslo/api/release.py | 9 ++++++--- aslo/api/tasks.py | 3 +-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/aslo/api/gh.py b/aslo/api/gh.py index de839eb..f94eef9 100644 --- a/aslo/api/gh.py +++ b/aslo/api/gh.py @@ -42,6 +42,5 @@ def find_tag_commit(repo_name, tag_name): return tag_commit -# No needed though :| (just a wrapper) def comment_on_commit(commit, message): commit.create_comment(message) diff --git a/aslo/api/release.py b/aslo/api/release.py index 3ed23aa..d2cbcaf 100644 --- a/aslo/api/release.py +++ b/aslo/api/release.py @@ -275,10 +275,13 @@ def handle_release(gh_json): repo_name = gh_json['repository']['name'] release = gh_json['release'] tag = release['tag_name'] + tag_commit = gh.find_tag_commit(gh_json['repository']['full_name'], tag) xo_asset = None - tag_commit = gh.find_tag_commit(gh_json['repository']['full_name'],tag) - #TODO: Extract message to constants file - gh.comment_on_commit(tag_commit,"Build has started :hourglass_flowing_sand:") + + # TODO: Extract message to constants file + gh.comment_on_commit(tag_commit, + "Build has started :hourglass_flowing_sand:") + if 'assets' in release and len(release['assets']) != 0: xo_asset = xo_file_exists(release['assets']) diff --git a/aslo/api/tasks.py b/aslo/api/tasks.py index a873086..41532ce 100644 --- a/aslo/api/tasks.py +++ b/aslo/api/tasks.py @@ -18,5 +18,4 @@ def release_process(self, gh_json): logger.info('Activity release process has finished successfully!') tag_commit = find_tag_commit( gh_json['repository']['full_name'], gh_json['release']['tag_name']) - comment_on_commit( - tag_commit, "Build Passed :white_check_mark: Download url will be provided soon :link:") + comment_on_commit(tag_commit, "Build Passed :white_check_mark:") From 50face4db42abdb03c63ce3bc652005808627906 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Thu, 13 Jul 2017 23:48:06 +0530 Subject: [PATCH 33/57] Add starting template --- aslo/web/templates/index.html | 252 ++++++++++++++++++++++++++++++---- 1 file changed, 224 insertions(+), 28 deletions(-) diff --git a/aslo/web/templates/index.html b/aslo/web/templates/index.html index a3ce364..1390a13 100644 --- a/aslo/web/templates/index.html +++ b/aslo/web/templates/index.html @@ -1,36 +1,232 @@ -{% extends "bootstrap/base.html" %} -{% block title %}Sugar Activities | SugarLabs{% endblock %} - -{% block navbar %} - -{% endblock %} -{% block content %} -{% endblock %} +
+
+
+ +
+ +
+

Turtle Arts

+ +
+ Loading image... +
+ +
+

Walter Benders

+
+ +
+ + +
+ +
+ +
+
+
+
+ +
+ +
+

Turtle Arts

+ +
+ Loading image... +
+ +
+

Walter Benders

+
+ +
+ + +
+ +
+ +
+
+
+
+ +
+ +
+

Turtle Arts

+ +
+ Loading image... +
+ +
+

Walter Benders

+
+ +
+ + +
+ +
+ +
+
+
+
+ +
+ +
+

Turtle Arts

+ +
+ Loading image... +
+ +
+

Walter Benders

+
+ +
+ + +
+ +
+ +
+
+
+
+ +
+ +
+

Turtle Arts

+ +
+ Loading image... +
+ +
+

Walter Benders

+
+ +
+ + +
+ +
+ +
+
+
+
+ +
+ +
+

Turtle Arts

+ +
+ Loading image... +
+ +
+

Walter Benders

+
+ +
+ + +
+ +
+ +
+
+
+ + + + + + + + + + + \ No newline at end of file From eba31c9bf7afaed6cc12c2ae0587686f771c1480 Mon Sep 17 00:00:00 2001 From: Jatin Dhankhar Date: Fri, 14 Jul 2017 14:35:53 +0530 Subject: [PATCH 34/57] Add detail template. Fix image asset issue --- aslo/web/__init__.py | 2 +- aslo/web/templates/index.html | 14 +++++++------- aslo/web/views.py | 6 +++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/aslo/web/__init__.py b/aslo/web/__init__.py index 20d9574..6826dc3 100644 --- a/aslo/web/__init__.py +++ b/aslo/web/__init__.py @@ -1,5 +1,5 @@ from flask import Blueprint -web = Blueprint('web', __name__, template_folder='templates') +web = Blueprint('web', __name__, template_folder='templates',static_folder='static') from . import views # noqa diff --git a/aslo/web/templates/index.html b/aslo/web/templates/index.html index 1390a13..c1bd9df 100644 --- a/aslo/web/templates/index.html +++ b/aslo/web/templates/index.html @@ -29,7 +29,7 @@ -