diff --git a/.gitignore b/.gitignore index 2e5ae26d..2ddff00a 100644 --- a/.gitignore +++ b/.gitignore @@ -118,6 +118,7 @@ sftp_pg_credential.env sftp_media_credential-dev.env sftp_pg_credential-dev.env inaware_credential.env +geonode_credential.env # apt-cacher-ng 71-apt-cacher-ng diff --git a/deployment/Makefile b/deployment/Makefile index 9fbd6d49..6167711a 100644 --- a/deployment/Makefile +++ b/deployment/Makefile @@ -494,3 +494,10 @@ load-flood-test-data: @echo "Load flood test data" @echo "------------------------------------------------------------------" @docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py loadfloodtestdata + +superuser: + @echo + @echo "------------------------------------------------------------------" + @echo "Creating a superuser in production mode" + @echo "------------------------------------------------------------------" + @docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py createsuperuser diff --git a/deployment/ansible/development/group_vars/all.sample.yml b/deployment/ansible/development/group_vars/all.sample.yml index d489326a..04407a70 100644 --- a/deployment/ansible/development/group_vars/all.sample.yml +++ b/deployment/ansible/development/group_vars/all.sample.yml @@ -118,6 +118,15 @@ inaware: user: inawareuser password: thepassword +# This is used to push hazard layer to GeoNode. +# Set this fact to your appropriate credentials. +geonode: + enable: False + user: test_geonode_user + password: test_geonode_password + url: http://url_to_geonode_instance + + # This declaration is used to describe port forwarding that is being used # by docker-compose. Leave it as default. docker_port_forward: diff --git a/deployment/ansible/development/group_vars/all.travis.yml b/deployment/ansible/development/group_vars/all.travis.yml index 66740b33..75ef65ca 100644 --- a/deployment/ansible/development/group_vars/all.travis.yml +++ b/deployment/ansible/development/group_vars/all.travis.yml @@ -118,6 +118,15 @@ inaware: user: password: +# This is used to push hazard layer to GeoNode. +# Set this fact to your appropriate credentials. +geonode: + enable: False + user: test_geonode_user + password: test_geonode_password + url: http://url_to_geonode_instance + + # This declaration is used to describe port forwarding that is being used # by docker-compose. Leave it as default. docker_port_forward: diff --git a/deployment/ansible/development/roles/docker_compose/tasks/main.yml b/deployment/ansible/development/roles/docker_compose/tasks/main.yml index fe262912..9503d4e4 100644 --- a/deployment/ansible/development/roles/docker_compose/tasks/main.yml +++ b/deployment/ansible/development/roles/docker_compose/tasks/main.yml @@ -26,3 +26,11 @@ owner: '{{ remote_user }}' group: '{{ remote_group }}' mode: "u=rw,g=rw,o=r" + +- name: customize geonode credentials + template: + src: geonode_credential.env.j2 + dest: '{{ project_path }}/deployment/geonode_credential.env' + owner: '{{ remote_user }}' + group: '{{ remote_group }}' + mode: "u=rw,g=rw,o=r" diff --git a/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 b/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 index b88358ae..fc03c6f3 100644 --- a/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 +++ b/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 @@ -76,6 +76,7 @@ services: - TASK_ALWAYS_EAGER={{ inasafe_django.task_always_eager }} - INASAFE_OUTPUT_DIR={{ inasafe_headless.working_dir }}/outputs - ON_TRAVIS={{ on_travis }} + - REALTIME_GEONODE_ENABLE={{ geonode.enable }} links: - smtp:smtp - db:db @@ -331,6 +332,8 @@ services: - INASAFE_WORK_DIR={{ inasafe_headless.working_dir }} - INASAFE_OUTPUT_DIR={{ inasafe_headless.working_dir }}/outputs - QGIS_DEBUG=0 + env_file: + - geonode_credential.env network_mode: "bridge" {% endif %} diff --git a/deployment/ansible/development/roles/docker_compose/templates/geonode_credential.env.j2 b/deployment/ansible/development/roles/docker_compose/templates/geonode_credential.env.j2 new file mode 100644 index 00000000..87c014b4 --- /dev/null +++ b/deployment/ansible/development/roles/docker_compose/templates/geonode_credential.env.j2 @@ -0,0 +1,6 @@ +# copy this template as sftp_credential.env for production mode credential +# and as sftp_credential-dev.env for development mode credential +REALTIME_GEONODE_ENABLE={{ geonode.enable }} +REALTIME_GEONODE_URL={{ geonode.url }} +REALTIME_GEONODE_USER={{ geonode.user }} +REALTIME_GEONODE_PASSWORD={{ geonode.password }} diff --git a/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 b/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 index 3824a661..5dc4d50a 100644 --- a/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 +++ b/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 @@ -8,6 +8,7 @@ + diff --git a/django_project/realtime/admin.py b/django_project/realtime/admin.py index b7dd4a4d..30628563 100644 --- a/django_project/realtime/admin.py +++ b/django_project/realtime/admin.py @@ -85,8 +85,9 @@ class EarthquakeReportInline(StackedInline): class EarthquakeAdmin(LeafletGeoAdmin): """Admin Class for Earthquake Model.""" - list_display = ('shake_id', 'source_type', 'time', 'location_description', - 'magnitude', 'depth') + list_display = ( + 'shake_id', 'source_type', 'time', 'location_description', + 'magnitude', 'depth', 'push_task_status', 'push_task_result') list_filter = ('location_description', ) search_fields = ['shake_id', 'location_description'] inlines = [ @@ -125,8 +126,9 @@ class FloodReportInline(StackedInline): class FloodAdmin(ModelAdmin): """Admin Class for Flood Event.""" - list_display = ('event_id', 'data_source', 'time', - 'total_affected', 'boundary_flooded') + list_display = ( + 'event_id', 'data_source', 'time', 'total_affected', + 'boundary_flooded', 'push_task_status', 'push_task_result') inlines = [ ImpactInline, @@ -151,9 +153,10 @@ class AshReportInline(StackedInline): class AshAdmin(ModelAdmin): """Admin class for Ash model""" - list_display = ('volcano', 'alert_level', 'event_time', - 'event_time_zone_string', 'eruption_height', - 'forecast_duration') + list_display = ( + 'volcano', 'alert_level', 'event_time', + 'event_time_zone_string', 'eruption_height', + 'forecast_duration', 'push_task_status', 'push_task_result') inlines = [ ImpactInline, AshReportInline diff --git a/django_project/realtime/app_settings.py b/django_project/realtime/app_settings.py index 100cfbe5..5265b4de 100644 --- a/django_project/realtime/app_settings.py +++ b/django_project/realtime/app_settings.py @@ -19,6 +19,8 @@ LOGGER_NAME = 'InaSAFE Realtime REST Server' ON_TRAVIS = ast.literal_eval(os.environ.get('ON_TRAVIS', 'False')) +REALTIME_GEONODE_ENABLE = ast.literal_eval( + os.environ.get('REALTIME_GEONODE_ENABLE', 'False')) # PROJECT_NAME: The project name for this apps e.g InaSAFE default_project_name = 'InaSAFE Realtime' diff --git a/django_project/realtime/migrations/0063_auto_20180604_0809.py b/django_project/realtime/migrations/0063_auto_20180604_0809.py new file mode 100644 index 00000000..252a57ce --- /dev/null +++ b/django_project/realtime/migrations/0063_auto_20180604_0809.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('realtime', '0062_auto_20180511_1932'), + ] + + operations = [ + migrations.AddField( + model_name='ash', + name='push_task_result', + field=models.TextField(default=b'', help_text='Task result of GeoNode Push Task', null=True, verbose_name='Report push task result', blank=True), + ), + migrations.AddField( + model_name='ash', + name='push_task_status', + field=models.CharField(default=None, max_length=255, blank=True, help_text='The Status for the GeoNode Push Task', null=True, verbose_name='GeoNode Push Task Status'), + ), + ] diff --git a/django_project/realtime/migrations/0064_auto_20180607_0724.py b/django_project/realtime/migrations/0064_auto_20180607_0724.py new file mode 100644 index 00000000..01afe89c --- /dev/null +++ b/django_project/realtime/migrations/0064_auto_20180607_0724.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('realtime', '0063_auto_20180604_0809'), + ] + + operations = [ + migrations.AddField( + model_name='flood', + name='push_task_result', + field=models.TextField(default=b'', help_text='Task result of GeoNode Push Task', null=True, verbose_name='Report push task result', blank=True), + ), + migrations.AddField( + model_name='flood', + name='push_task_status', + field=models.CharField(default=None, max_length=255, blank=True, help_text='The Status for the GeoNode Push Task', null=True, verbose_name='GeoNode Push Task Status'), + ), + ] diff --git a/django_project/realtime/migrations/0065_auto_20180607_0742.py b/django_project/realtime/migrations/0065_auto_20180607_0742.py new file mode 100644 index 00000000..e2a820be --- /dev/null +++ b/django_project/realtime/migrations/0065_auto_20180607_0742.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('realtime', '0064_auto_20180607_0724'), + ] + + operations = [ + migrations.AddField( + model_name='earthquake', + name='push_task_result', + field=models.TextField(default=b'', help_text='Task result of GeoNode Push Task', null=True, verbose_name='Report push task result', blank=True), + ), + migrations.AddField( + model_name='earthquake', + name='push_task_status', + field=models.CharField(default=None, max_length=255, blank=True, help_text='The Status for the GeoNode Push Task', null=True, verbose_name='GeoNode Push Task Status'), + ), + ] diff --git a/django_project/realtime/models/ash.py b/django_project/realtime/models/ash.py index d83c0a37..c924917c 100644 --- a/django_project/realtime/models/ash.py +++ b/django_project/realtime/models/ash.py @@ -85,6 +85,21 @@ class Meta: default='None', blank=True) + push_task_status = models.CharField( + verbose_name=_('GeoNode Push Task Status'), + help_text=_('The Status for the GeoNode Push Task'), + max_length=255, + default=None, + null=True, + blank=True) + + push_task_result = models.TextField( + verbose_name=_('Report push task result'), + help_text=_('Task result of GeoNode Push Task'), + default='', + blank=True, + null=True) + objects = models.GeoManager() def __unicode__(self): diff --git a/django_project/realtime/models/earthquake.py b/django_project/realtime/models/earthquake.py index 72bf3602..1b50e2e8 100644 --- a/django_project/realtime/models/earthquake.py +++ b/django_project/realtime/models/earthquake.py @@ -117,6 +117,21 @@ class Meta: ), default=False) + push_task_status = models.CharField( + verbose_name=_('GeoNode Push Task Status'), + help_text=_('The Status for the GeoNode Push Task'), + max_length=255, + default=None, + null=True, + blank=True) + + push_task_result = models.TextField( + verbose_name=_('Report push task result'), + help_text=_('Task result of GeoNode Push Task'), + default='', + blank=True, + null=True) + objects = EarthquakeManager() def __unicode__(self): diff --git a/django_project/realtime/models/flood.py b/django_project/realtime/models/flood.py index 48cfc57f..a05ccbaf 100644 --- a/django_project/realtime/models/flood.py +++ b/django_project/realtime/models/flood.py @@ -152,6 +152,21 @@ class Meta: help_text=_('Total boundary affected by flood'), default=0) + push_task_status = models.CharField( + verbose_name=_('GeoNode Push Task Status'), + help_text=_('The Status for the GeoNode Push Task'), + max_length=255, + default=None, + null=True, + blank=True) + + push_task_result = models.TextField( + verbose_name=_('Report push task result'), + help_text=_('Task result of GeoNode Push Task'), + default='', + blank=True, + null=True) + objects = FloodManager() def delete(self, using=None): diff --git a/django_project/realtime/signals/ash.py b/django_project/realtime/signals/ash.py index 11fd2463..030c77d8 100644 --- a/django_project/realtime/signals/ash.py +++ b/django_project/realtime/signals/ash.py @@ -7,6 +7,7 @@ from realtime.app_settings import LOGGER_NAME, ANALYSIS_LANGUAGES from realtime.models.ash import Ash from realtime.tasks.ash import generate_event_report, generate_hazard_layer +from realtime.tasks.geonode import push_hazard_to_geonode __author__ = 'Rizky Maulana Nugraha ' __date__ = '7/18/16' @@ -37,5 +38,7 @@ def ash_post_save(sender, instance, **kwargs): if instance.analysis_flag: for lang in ANALYSIS_LANGUAGES: generate_event_report.delay(instance, locale=lang) + push_hazard_to_geonode.delay(sender, instance) + except BaseException as e: LOGGER.exception(e) diff --git a/django_project/realtime/signals/earthquake.py b/django_project/realtime/signals/earthquake.py index aa5306be..46e18983 100644 --- a/django_project/realtime/signals/earthquake.py +++ b/django_project/realtime/signals/earthquake.py @@ -7,6 +7,7 @@ from realtime.app_settings import LOGGER_NAME, ANALYSIS_LANGUAGES from realtime.models.earthquake import Earthquake from realtime.tasks.earthquake import generate_event_report +from realtime.tasks.geonode import push_hazard_to_geonode __author__ = 'Rizky Maulana Nugraha ' __date__ = '7/18/16' @@ -31,5 +32,6 @@ def earthquake_post_save(sender, instance, **kwargs): for lang in ANALYSIS_LANGUAGES: generate_event_report.delay( instance, locale=lang) + push_hazard_to_geonode.delay(sender, instance) except BaseException: pass diff --git a/django_project/realtime/signals/flood.py b/django_project/realtime/signals/flood.py index 4f07570f..e8e28da5 100644 --- a/django_project/realtime/signals/flood.py +++ b/django_project/realtime/signals/flood.py @@ -7,6 +7,7 @@ from realtime.app_settings import LOGGER_NAME, ANALYSIS_LANGUAGES from realtime.models.flood import Flood from realtime.tasks.flood import generate_event_report +from realtime.tasks.geonode import push_hazard_to_geonode __author__ = 'Rizky Maulana Nugraha ' __date__ = '12/4/15' @@ -30,5 +31,7 @@ def flood_post_save( if instance.analysis_flag: for lang in ANALYSIS_LANGUAGES: generate_event_report.delay(instance, locale=lang) + if instance.analysis_flag: + push_hazard_to_geonode.delay(instance) except Exception as e: LOGGER.exception(e) diff --git a/django_project/realtime/tasks/__init__.py b/django_project/realtime/tasks/__init__.py index 41367db0..3851c80d 100644 --- a/django_project/realtime/tasks/__init__.py +++ b/django_project/realtime/tasks/__init__.py @@ -6,6 +6,7 @@ from realtime.tasks.flood import * # noqa from realtime.tasks.ash import * # noqa from realtime.tasks.indicator import * # noqa +from realtime.tasks.geonode import * # noqa __author__ = 'Rizky Maulana Nugraha ' __date__ = '12/3/15' diff --git a/django_project/realtime/tasks/ash.py b/django_project/realtime/tasks/ash.py index 8a56dff1..85632d39 100644 --- a/django_project/realtime/tasks/ash.py +++ b/django_project/realtime/tasks/ash.py @@ -73,7 +73,8 @@ def generate_hazard_layer(ash_event): # Handle hazard process handle_hazard_process.s( event_id=ash_event.id - ).set(queue=handle_hazard_process.queue) + ).set(queue=handle_hazard_process.queue), + ) @app.task diff --git a/django_project/realtime/tasks/geonode.py b/django_project/realtime/tasks/geonode.py new file mode 100644 index 00000000..335c6bcb --- /dev/null +++ b/django_project/realtime/tasks/geonode.py @@ -0,0 +1,87 @@ +# coding=utf-8 +"""InaSAFE Django task related to GeoNode upload.""" +from __future__ import absolute_import +import logging + +from celery import chain + +from core.celery_app import app + +from realtime.models.ash import Ash +from realtime.models.flood import Flood +from realtime.models.earthquake import Earthquake +from realtime.app_settings import LOGGER_NAME, REALTIME_GEONODE_ENABLE +from realtime.tasks.headless.inasafe_wrapper import push_to_geonode + +LOGGER = logging.getLogger(LOGGER_NAME) +GEONODE_PUSH_SUCCESS = 0 + + +@app.task(queue='inasafe-django') +def handle_push_to_geonode(push_result, hazard_class_name, hazard_event_id): + """Handle geonode push result.""" + hazard_class_mapping = { + Ash.__name__: Ash, + Flood.__name__: Flood, + Earthquake.__name__: Earthquake + } + + # Hacky thing to get proper class + if hazard_class_name not in hazard_class_mapping.keys(): + for key in hazard_class_mapping.keys(): + if key.lower() in hazard_class_name.lower(): + hazard_class_name = key + break + hazard_class = hazard_class_mapping.get(hazard_class_name) + task_state = 'FAILURE' + if not push_result: + task_state = 'FAILURE' + elif push_result['status'] == GEONODE_PUSH_SUCCESS: + task_state = 'SUCCESS' + + # Use update so it doesn't trigger save signal + hazard_class.objects.filter(id=hazard_event_id).update( + push_task_status=task_state, + push_task_result=push_result + ) + + +@app.task(queue='inasafe-django') +def push_hazard_to_geonode(hazard_event): + """Upload layer to geonode and update the status of hazard.""" + # If geonode push is disabled, skip the task + if not REALTIME_GEONODE_ENABLE: + hazard_event.__class__.objects.filter(id=hazard_event.id).update( + push_task_status='DISABLED', + push_task_result='GeoNode push is disabled in the setting.' + ) + return + LOGGER.info('Push layer to geonode.') + # Skip if it's already running. + if hazard_event.push_task_status: + return + hazard_layer_uri = hazard_event.hazard_path + hazard_event_id = hazard_event.id + hazard_class = hazard_event.__class__ + hazard_class_name = hazard_class.__name__ + + tasks_chain = chain( + # Push to layer to geonode + push_to_geonode.s( + hazard_layer_uri + ).set(queue=push_to_geonode.queue), + + # Handle the push result + handle_push_to_geonode.s( + hazard_class_name, + hazard_event_id + ).set(queue=handle_push_to_geonode.queue) + ) + + @app.task + def _handle_error(req, exc, traceback): + """Update task status as Failure.""" + hazard_event.push_task_status = 'FAILURE' + + async_result = tasks_chain.apply_async() + hazard_event.push_task_status = async_result.state diff --git a/django_project/realtime/tasks/headless/celeryconfig.py b/django_project/realtime/tasks/headless/celeryconfig.py index 2e6e3123..67764553 100644 --- a/django_project/realtime/tasks/headless/celeryconfig.py +++ b/django_project/realtime/tasks/headless/celeryconfig.py @@ -39,6 +39,9 @@ 'inasafe.headless.tasks.check_broker_connection': { 'queue': 'inasafe-headless' }, + 'inasafe.headless.tasks.push_to_geonode': { + 'queue': 'inasafe-headless-geonode' + } } # RMN: This is really important. diff --git a/django_project/realtime/tasks/headless/inasafe_wrapper.py b/django_project/realtime/tasks/headless/inasafe_wrapper.py index f19d757a..414a4c34 100644 --- a/django_project/realtime/tasks/headless/inasafe_wrapper.py +++ b/django_project/realtime/tasks/headless/inasafe_wrapper.py @@ -250,3 +250,11 @@ def check_broker_connection(): """ LOGGER.info('proxy tasks') return True + + +@app.task( + name='inasafe.headless.tasks.push_to_geonode', + queue='inasafe-headless') +def push_to_geonode(layer_uri): + LOGGER.info('proxy tasks') + pass diff --git a/django_project/realtime/tasks/test/test_geonode_task.py b/django_project/realtime/tasks/test/test_geonode_task.py new file mode 100644 index 00000000..e86c98c8 --- /dev/null +++ b/django_project/realtime/tasks/test/test_geonode_task.py @@ -0,0 +1,20 @@ +# coding=utf-8 +import os +from unittest import skip +from django.test import TestCase + +from realtime.tasks.geonode import push_hazard_to_geonode +from realtime.models.flood import Flood +from realtime.tests.model_factories import FloodFactory + + +class TestModelFlood(TestCase): + + @skip('Under development') + def test_upload_flood_hazard(self): + flood = FloodFactory.create() + message = 'The flood object is instantiated successfully.' + self.assertIsNotNone(flood.id, message) + self.assertTrue(os.path.exists(flood.hazard_path)) + async_result = push_hazard_to_geonode.delay(Flood, flood) + _ = async_result.get() # noqa diff --git a/django_project/realtime/tasks/test/test_headless_task.py b/django_project/realtime/tasks/test/test_headless_task.py index 305bb80f..0a7a32ec 100644 --- a/django_project/realtime/tasks/test/test_headless_task.py +++ b/django_project/realtime/tasks/test/test_headless_task.py @@ -13,7 +13,8 @@ ASH_LAYER_ORDER, FLOOD_EXPOSURE, FLOOD_REPORT_TEMPLATE_EN, - FLOOD_LAYER_ORDER + FLOOD_LAYER_ORDER, + REALTIME_GEONODE_ENABLE, ) from realtime.tasks.headless.celery_app import app as headless_app from realtime.tasks.headless.inasafe_wrapper import ( @@ -24,6 +25,7 @@ generate_report, get_generated_report, check_broker_connection, + push_to_geonode, ) from realtime.utils import celery_worker_connected @@ -86,8 +88,7 @@ def test_get_keywords(self): result = get_keywords.delay(place_layer_uri) keywords = result.get() self.assertIsNotNone(keywords) - self.assertEqual( - keywords['layer_purpose'], 'exposure') + self.assertEqual(keywords['layer_purpose'], 'exposure') self.assertEqual(keywords['exposure'], 'place') self.assertTrue(os.path.exists(earthquake_layer_uri)) @@ -456,3 +457,17 @@ def test_flood_analysis_real_exposure(self): # Check if the default map reports are not found self.assertNotIn('inasafe-map-report-portrait', product_keys) self.assertNotIn('inasafe-map-report-landscape', product_keys) + + @unittest.skipIf(not REALTIME_GEONODE_ENABLE, 'GeoNode push is disabled.') + def test_push_tif_to_geonode(self): + """Test push tif layer to geonode functionality.""" + async_result = push_to_geonode.delay(shakemap_layer_uri) + result = async_result.get() + self.assertEqual(result['status'], 0, result['message']) + + @unittest.skipIf(not REALTIME_GEONODE_ENABLE, 'GeoNode push is disabled.') + def test_push_geojson_to_geonode(self): + """Test push geojson layer to geonode functionality.""" + async_result = push_to_geonode.delay(flood_layer_uri) + result = async_result.get() + self.assertEqual(result['status'], 0, result['message'])