From 9718dfe5d3dfa177f741cef3b2bf05656761cd37 Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Thu, 3 May 2018 21:43:29 +0700 Subject: [PATCH 01/15] Add credential setting for geonode. --- .gitignore | 1 + deployment/ansible/development/group_vars/all.sample.yml | 8 ++++++++ deployment/ansible/development/group_vars/all.travis.yml | 8 ++++++++ .../development/roles/docker_compose/tasks/main.yml | 8 ++++++++ .../templates/docker-compose.override.yml.j2 | 2 ++ .../docker_compose/templates/geonode_credential.env.j2 | 5 +++++ 6 files changed, 32 insertions(+) create mode 100644 deployment/ansible/development/roles/docker_compose/templates/geonode_credential.env.j2 diff --git a/.gitignore b/.gitignore index 2e5ae26d..2ddff00a 100644 --- a/.gitignore +++ b/.gitignore @@ -118,6 +118,7 @@ sftp_pg_credential.env sftp_media_credential-dev.env sftp_pg_credential-dev.env inaware_credential.env +geonode_credential.env # apt-cacher-ng 71-apt-cacher-ng diff --git a/deployment/ansible/development/group_vars/all.sample.yml b/deployment/ansible/development/group_vars/all.sample.yml index d489326a..0be99bc2 100644 --- a/deployment/ansible/development/group_vars/all.sample.yml +++ b/deployment/ansible/development/group_vars/all.sample.yml @@ -118,6 +118,14 @@ inaware: user: inawareuser password: thepassword +# This is used to push hazard layer to GeoNode. +# Set this fact to your appropriate credentials. If not used, +# leave as blank +geonode: + host: http://138.201.158.66 + user: test_geonode_user + password: test_geonode_password + # This declaration is used to describe port forwarding that is being used # by docker-compose. Leave it as default. docker_port_forward: diff --git a/deployment/ansible/development/group_vars/all.travis.yml b/deployment/ansible/development/group_vars/all.travis.yml index 66740b33..21345128 100644 --- a/deployment/ansible/development/group_vars/all.travis.yml +++ b/deployment/ansible/development/group_vars/all.travis.yml @@ -118,6 +118,14 @@ inaware: user: password: +# This is used to push hazard layer to GeoNode. +# Set this fact to your appropriate credentials. If not used, +# leave as blank +geonode: + host: + user: + password: + # This declaration is used to describe port forwarding that is being used # by docker-compose. Leave it as default. docker_port_forward: diff --git a/deployment/ansible/development/roles/docker_compose/tasks/main.yml b/deployment/ansible/development/roles/docker_compose/tasks/main.yml index fe262912..9503d4e4 100644 --- a/deployment/ansible/development/roles/docker_compose/tasks/main.yml +++ b/deployment/ansible/development/roles/docker_compose/tasks/main.yml @@ -26,3 +26,11 @@ owner: '{{ remote_user }}' group: '{{ remote_group }}' mode: "u=rw,g=rw,o=r" + +- name: customize geonode credentials + template: + src: geonode_credential.env.j2 + dest: '{{ project_path }}/deployment/geonode_credential.env' + owner: '{{ remote_user }}' + group: '{{ remote_group }}' + mode: "u=rw,g=rw,o=r" diff --git a/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 b/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 index b88358ae..596f7f50 100644 --- a/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 +++ b/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 @@ -331,6 +331,8 @@ services: - INASAFE_WORK_DIR={{ inasafe_headless.working_dir }} - INASAFE_OUTPUT_DIR={{ inasafe_headless.working_dir }}/outputs - QGIS_DEBUG=0 + env_file: + - geonode_credential.env network_mode: "bridge" {% endif %} diff --git a/deployment/ansible/development/roles/docker_compose/templates/geonode_credential.env.j2 b/deployment/ansible/development/roles/docker_compose/templates/geonode_credential.env.j2 new file mode 100644 index 00000000..11ca1aa4 --- /dev/null +++ b/deployment/ansible/development/roles/docker_compose/templates/geonode_credential.env.j2 @@ -0,0 +1,5 @@ +# copy this template as sftp_credential.env for production mode credential +# and as sftp_credential-dev.env for development mode credential +REALTIME_GEONODE_URL={{ geonode.host }} +REALTIME_GEONODE_USER={{ geonode.user }} +REALTIME_GEONODE_PASSWORD={{ geonode.password }} From 19b2f767b7667879598d5093c4c3edfbe0239f34 Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Thu, 3 May 2018 22:24:45 +0700 Subject: [PATCH 02/15] Add celery setup for push to geonode tasks. --- .../realtime/tasks/headless/celeryconfig.py | 3 +++ .../realtime/tasks/headless/inasafe_wrapper.py | 8 ++++++++ .../realtime/tasks/test/test_headless_task.py | 11 +++++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/django_project/realtime/tasks/headless/celeryconfig.py b/django_project/realtime/tasks/headless/celeryconfig.py index 2e6e3123..67764553 100644 --- a/django_project/realtime/tasks/headless/celeryconfig.py +++ b/django_project/realtime/tasks/headless/celeryconfig.py @@ -39,6 +39,9 @@ 'inasafe.headless.tasks.check_broker_connection': { 'queue': 'inasafe-headless' }, + 'inasafe.headless.tasks.push_to_geonode': { + 'queue': 'inasafe-headless-geonode' + } } # RMN: This is really important. diff --git a/django_project/realtime/tasks/headless/inasafe_wrapper.py b/django_project/realtime/tasks/headless/inasafe_wrapper.py index f19d757a..414a4c34 100644 --- a/django_project/realtime/tasks/headless/inasafe_wrapper.py +++ b/django_project/realtime/tasks/headless/inasafe_wrapper.py @@ -250,3 +250,11 @@ def check_broker_connection(): """ LOGGER.info('proxy tasks') return True + + +@app.task( + name='inasafe.headless.tasks.push_to_geonode', + queue='inasafe-headless') +def push_to_geonode(layer_uri): + LOGGER.info('proxy tasks') + pass diff --git a/django_project/realtime/tasks/test/test_headless_task.py b/django_project/realtime/tasks/test/test_headless_task.py index 305bb80f..88ab2635 100644 --- a/django_project/realtime/tasks/test/test_headless_task.py +++ b/django_project/realtime/tasks/test/test_headless_task.py @@ -24,6 +24,7 @@ generate_report, get_generated_report, check_broker_connection, + push_to_geonode, ) from realtime.utils import celery_worker_connected @@ -86,8 +87,7 @@ def test_get_keywords(self): result = get_keywords.delay(place_layer_uri) keywords = result.get() self.assertIsNotNone(keywords) - self.assertEqual( - keywords['layer_purpose'], 'exposure') + self.assertEqual(keywords['layer_purpose'], 'exposure') self.assertEqual(keywords['exposure'], 'place') self.assertTrue(os.path.exists(earthquake_layer_uri)) @@ -456,3 +456,10 @@ def test_flood_analysis_real_exposure(self): # Check if the default map reports are not found self.assertNotIn('inasafe-map-report-portrait', product_keys) self.assertNotIn('inasafe-map-report-landscape', product_keys) + + @unittest.skipIf(os.environ.get('ON_TRAVIS', False), 'No geonode instance') + def test_push_tif_to_geonode(self): + """Test push tif layer to geonode functionality.""" + async_result = push_to_geonode.delay(shakemap_layer_uri) + result = async_result.get() + self.assertEqual(result['status'], 0, result['message']) From 7f848309a5e573709007597d46ec580264d22227 Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Mon, 4 Jun 2018 14:29:42 +0700 Subject: [PATCH 03/15] Update geonode variable in yml. --- deployment/ansible/development/group_vars/all.sample.yml | 4 +++- deployment/ansible/development/group_vars/all.travis.yml | 6 ++++-- .../docker_compose/templates/geonode_credential.env.j2 | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/deployment/ansible/development/group_vars/all.sample.yml b/deployment/ansible/development/group_vars/all.sample.yml index 0be99bc2..43d049cf 100644 --- a/deployment/ansible/development/group_vars/all.sample.yml +++ b/deployment/ansible/development/group_vars/all.sample.yml @@ -122,9 +122,11 @@ inaware: # Set this fact to your appropriate credentials. If not used, # leave as blank geonode: - host: http://138.201.158.66 + enable: False user: test_geonode_user password: test_geonode_password + url: http://url_to_geonode_instance + # This declaration is used to describe port forwarding that is being used # by docker-compose. Leave it as default. diff --git a/deployment/ansible/development/group_vars/all.travis.yml b/deployment/ansible/development/group_vars/all.travis.yml index 21345128..8f922c8e 100644 --- a/deployment/ansible/development/group_vars/all.travis.yml +++ b/deployment/ansible/development/group_vars/all.travis.yml @@ -121,10 +121,12 @@ inaware: # This is used to push hazard layer to GeoNode. # Set this fact to your appropriate credentials. If not used, # leave as blank -geonode: - host: +geonode_push: + enable: False user: password: + url: + # This declaration is used to describe port forwarding that is being used # by docker-compose. Leave it as default. diff --git a/deployment/ansible/development/roles/docker_compose/templates/geonode_credential.env.j2 b/deployment/ansible/development/roles/docker_compose/templates/geonode_credential.env.j2 index 11ca1aa4..87c014b4 100644 --- a/deployment/ansible/development/roles/docker_compose/templates/geonode_credential.env.j2 +++ b/deployment/ansible/development/roles/docker_compose/templates/geonode_credential.env.j2 @@ -1,5 +1,6 @@ # copy this template as sftp_credential.env for production mode credential # and as sftp_credential-dev.env for development mode credential -REALTIME_GEONODE_URL={{ geonode.host }} +REALTIME_GEONODE_ENABLE={{ geonode.enable }} +REALTIME_GEONODE_URL={{ geonode.url }} REALTIME_GEONODE_USER={{ geonode.user }} REALTIME_GEONODE_PASSWORD={{ geonode.password }} From 5bc7c85da4edc5cf332d020792bbe8a01410449a Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Wed, 6 Jun 2018 16:24:45 +0700 Subject: [PATCH 04/15] Add upload to geonode in task chain. --- .../migrations/0063_auto_20180604_0809.py | 24 ++++++++++++++ django_project/realtime/models/ash.py | 15 +++++++++ django_project/realtime/tasks/__init__.py | 1 + django_project/realtime/tasks/ash.py | 14 ++++++++- django_project/realtime/tasks/geonode.py | 31 +++++++++++++++++++ 5 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 django_project/realtime/migrations/0063_auto_20180604_0809.py create mode 100644 django_project/realtime/tasks/geonode.py diff --git a/django_project/realtime/migrations/0063_auto_20180604_0809.py b/django_project/realtime/migrations/0063_auto_20180604_0809.py new file mode 100644 index 00000000..252a57ce --- /dev/null +++ b/django_project/realtime/migrations/0063_auto_20180604_0809.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('realtime', '0062_auto_20180511_1932'), + ] + + operations = [ + migrations.AddField( + model_name='ash', + name='push_task_result', + field=models.TextField(default=b'', help_text='Task result of GeoNode Push Task', null=True, verbose_name='Report push task result', blank=True), + ), + migrations.AddField( + model_name='ash', + name='push_task_status', + field=models.CharField(default=None, max_length=255, blank=True, help_text='The Status for the GeoNode Push Task', null=True, verbose_name='GeoNode Push Task Status'), + ), + ] diff --git a/django_project/realtime/models/ash.py b/django_project/realtime/models/ash.py index d83c0a37..c924917c 100644 --- a/django_project/realtime/models/ash.py +++ b/django_project/realtime/models/ash.py @@ -85,6 +85,21 @@ class Meta: default='None', blank=True) + push_task_status = models.CharField( + verbose_name=_('GeoNode Push Task Status'), + help_text=_('The Status for the GeoNode Push Task'), + max_length=255, + default=None, + null=True, + blank=True) + + push_task_result = models.TextField( + verbose_name=_('Report push task result'), + help_text=_('Task result of GeoNode Push Task'), + default='', + blank=True, + null=True) + objects = models.GeoManager() def __unicode__(self): diff --git a/django_project/realtime/tasks/__init__.py b/django_project/realtime/tasks/__init__.py index 41367db0..3851c80d 100644 --- a/django_project/realtime/tasks/__init__.py +++ b/django_project/realtime/tasks/__init__.py @@ -6,6 +6,7 @@ from realtime.tasks.flood import * # noqa from realtime.tasks.ash import * # noqa from realtime.tasks.indicator import * # noqa +from realtime.tasks.geonode import * # noqa __author__ = 'Rizky Maulana Nugraha ' __date__ = '12/3/15' diff --git a/django_project/realtime/tasks/ash.py b/django_project/realtime/tasks/ash.py index 8a56dff1..8b9cdbde 100644 --- a/django_project/realtime/tasks/ash.py +++ b/django_project/realtime/tasks/ash.py @@ -21,6 +21,8 @@ from realtime.tasks.realtime.ash import process_ash from realtime.utils import substitute_layer_order, template_names, \ template_paths +from realtime.tasks.geonode import ( + upload_layer_to_geonode, handle_push_to_geonode) __author__ = 'Rizky Maulana Nugraha ' __date__ = '20/7/17' @@ -73,7 +75,17 @@ def generate_hazard_layer(ash_event): # Handle hazard process handle_hazard_process.s( event_id=ash_event.id - ).set(queue=handle_hazard_process.queue) + ).set(queue=handle_hazard_process.queue), + + # # Push to GeoNode + upload_layer_to_geonode.s( + event_id=ash_event.id + ).set(queue=upload_layer_to_geonode.queue), + # + # # Handle push to geonode process + handle_push_to_geonode.s( + event_id=ash_event.id + ).set(queue=handle_push_to_geonode.queue) ) @app.task diff --git a/django_project/realtime/tasks/geonode.py b/django_project/realtime/tasks/geonode.py new file mode 100644 index 00000000..849cb59c --- /dev/null +++ b/django_project/realtime/tasks/geonode.py @@ -0,0 +1,31 @@ +# coding=utf-8 +"""InaSAFE Django task related to GeoNode upload.""" +from __future__ import absolute_import + +from core.celery_app import app + +from realtime.models.ash import Ash +from realtime.tasks.headless.inasafe_wrapper import push_to_geonode + +GEONODE_PUSH_SUCCESS = 0 + +@app.task(queue='inasafe-django') +def upload_layer_to_geonode(process_result, event_id): + """Upload layer to geonode.""" + hazard = Ash.objects.get(id=event_id) + hazard_layer_uri = hazard.hazard_path + return push_to_geonode(hazard_layer_uri) + +@app.task(queue='inasafe-django') +def handle_push_to_geonode(push_result, event_id): + """Handle geonode push result.""" + # TODO(IS): make it generic for all hazard, somehow + hazard = Ash.objects.get(id=event_id) + task_state = 'FAILURE' + if not push_result: + task_state = 'FAILURE' + elif push_result['status'] == GEONODE_PUSH_SUCCESS: + task_state = 'SUCCESS' + hazard.push_task_status = task_state + hazard.push_task_result = push_result + hazard.save() From 6402f4dcc7512667f29d3909514b0ef7b85f638b Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Thu, 7 Jun 2018 13:34:41 +0700 Subject: [PATCH 05/15] Set dummy geonode credentials. --- deployment/ansible/development/group_vars/all.travis.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/deployment/ansible/development/group_vars/all.travis.yml b/deployment/ansible/development/group_vars/all.travis.yml index 8f922c8e..c567dfcd 100644 --- a/deployment/ansible/development/group_vars/all.travis.yml +++ b/deployment/ansible/development/group_vars/all.travis.yml @@ -119,13 +119,12 @@ inaware: password: # This is used to push hazard layer to GeoNode. -# Set this fact to your appropriate credentials. If not used, -# leave as blank +# Set this fact to your appropriate credentials. geonode_push: enable: False - user: - password: - url: + user: InvalidGeoNodeUser + password: InvalidGeoNodePassword + url: InvalidGeoNodeURL # This declaration is used to describe port forwarding that is being used From cba7f4ffcc6a5dee9043350c21e0923bedc3b8d5 Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Thu, 7 Jun 2018 13:36:07 +0700 Subject: [PATCH 06/15] Set dummy geonode credentials. --- deployment/ansible/development/group_vars/all.sample.yml | 3 +-- deployment/ansible/development/group_vars/all.travis.yml | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/deployment/ansible/development/group_vars/all.sample.yml b/deployment/ansible/development/group_vars/all.sample.yml index 43d049cf..04407a70 100644 --- a/deployment/ansible/development/group_vars/all.sample.yml +++ b/deployment/ansible/development/group_vars/all.sample.yml @@ -119,8 +119,7 @@ inaware: password: thepassword # This is used to push hazard layer to GeoNode. -# Set this fact to your appropriate credentials. If not used, -# leave as blank +# Set this fact to your appropriate credentials. geonode: enable: False user: test_geonode_user diff --git a/deployment/ansible/development/group_vars/all.travis.yml b/deployment/ansible/development/group_vars/all.travis.yml index c567dfcd..75ef65ca 100644 --- a/deployment/ansible/development/group_vars/all.travis.yml +++ b/deployment/ansible/development/group_vars/all.travis.yml @@ -120,11 +120,11 @@ inaware: # This is used to push hazard layer to GeoNode. # Set this fact to your appropriate credentials. -geonode_push: +geonode: enable: False - user: InvalidGeoNodeUser - password: InvalidGeoNodePassword - url: InvalidGeoNodeURL + user: test_geonode_user + password: test_geonode_password + url: http://url_to_geonode_instance # This declaration is used to describe port forwarding that is being used From f3369c2316ea1aa218c57c861c824719d9d5138f Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Thu, 7 Jun 2018 14:45:46 +0700 Subject: [PATCH 07/15] Add migration for flood and EQ for geonode push. --- .../migrations/0064_auto_20180607_0724.py | 24 +++++++++++++++++++ .../migrations/0065_auto_20180607_0742.py | 24 +++++++++++++++++++ django_project/realtime/models/earthquake.py | 15 ++++++++++++ django_project/realtime/models/flood.py | 15 ++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 django_project/realtime/migrations/0064_auto_20180607_0724.py create mode 100644 django_project/realtime/migrations/0065_auto_20180607_0742.py diff --git a/django_project/realtime/migrations/0064_auto_20180607_0724.py b/django_project/realtime/migrations/0064_auto_20180607_0724.py new file mode 100644 index 00000000..01afe89c --- /dev/null +++ b/django_project/realtime/migrations/0064_auto_20180607_0724.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('realtime', '0063_auto_20180604_0809'), + ] + + operations = [ + migrations.AddField( + model_name='flood', + name='push_task_result', + field=models.TextField(default=b'', help_text='Task result of GeoNode Push Task', null=True, verbose_name='Report push task result', blank=True), + ), + migrations.AddField( + model_name='flood', + name='push_task_status', + field=models.CharField(default=None, max_length=255, blank=True, help_text='The Status for the GeoNode Push Task', null=True, verbose_name='GeoNode Push Task Status'), + ), + ] diff --git a/django_project/realtime/migrations/0065_auto_20180607_0742.py b/django_project/realtime/migrations/0065_auto_20180607_0742.py new file mode 100644 index 00000000..e2a820be --- /dev/null +++ b/django_project/realtime/migrations/0065_auto_20180607_0742.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('realtime', '0064_auto_20180607_0724'), + ] + + operations = [ + migrations.AddField( + model_name='earthquake', + name='push_task_result', + field=models.TextField(default=b'', help_text='Task result of GeoNode Push Task', null=True, verbose_name='Report push task result', blank=True), + ), + migrations.AddField( + model_name='earthquake', + name='push_task_status', + field=models.CharField(default=None, max_length=255, blank=True, help_text='The Status for the GeoNode Push Task', null=True, verbose_name='GeoNode Push Task Status'), + ), + ] diff --git a/django_project/realtime/models/earthquake.py b/django_project/realtime/models/earthquake.py index 72bf3602..1b50e2e8 100644 --- a/django_project/realtime/models/earthquake.py +++ b/django_project/realtime/models/earthquake.py @@ -117,6 +117,21 @@ class Meta: ), default=False) + push_task_status = models.CharField( + verbose_name=_('GeoNode Push Task Status'), + help_text=_('The Status for the GeoNode Push Task'), + max_length=255, + default=None, + null=True, + blank=True) + + push_task_result = models.TextField( + verbose_name=_('Report push task result'), + help_text=_('Task result of GeoNode Push Task'), + default='', + blank=True, + null=True) + objects = EarthquakeManager() def __unicode__(self): diff --git a/django_project/realtime/models/flood.py b/django_project/realtime/models/flood.py index 48cfc57f..a05ccbaf 100644 --- a/django_project/realtime/models/flood.py +++ b/django_project/realtime/models/flood.py @@ -152,6 +152,21 @@ class Meta: help_text=_('Total boundary affected by flood'), default=0) + push_task_status = models.CharField( + verbose_name=_('GeoNode Push Task Status'), + help_text=_('The Status for the GeoNode Push Task'), + max_length=255, + default=None, + null=True, + blank=True) + + push_task_result = models.TextField( + verbose_name=_('Report push task result'), + help_text=_('Task result of GeoNode Push Task'), + default='', + blank=True, + null=True) + objects = FloodManager() def delete(self, using=None): From 02f9d3196c18235ea7574464eb8c15a62e592536 Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Thu, 7 Jun 2018 14:46:07 +0700 Subject: [PATCH 08/15] Call geonode task from save signal instead. --- django_project/realtime/signals/ash.py | 3 ++ django_project/realtime/signals/earthquake.py | 2 + django_project/realtime/signals/flood.py | 3 ++ django_project/realtime/tasks/ash.py | 11 ----- django_project/realtime/tasks/geonode.py | 45 +++++++++++++++---- .../realtime/tasks/test/test_geonode_task.py | 20 +++++++++ 6 files changed, 64 insertions(+), 20 deletions(-) create mode 100644 django_project/realtime/tasks/test/test_geonode_task.py diff --git a/django_project/realtime/signals/ash.py b/django_project/realtime/signals/ash.py index 11fd2463..030c77d8 100644 --- a/django_project/realtime/signals/ash.py +++ b/django_project/realtime/signals/ash.py @@ -7,6 +7,7 @@ from realtime.app_settings import LOGGER_NAME, ANALYSIS_LANGUAGES from realtime.models.ash import Ash from realtime.tasks.ash import generate_event_report, generate_hazard_layer +from realtime.tasks.geonode import push_hazard_to_geonode __author__ = 'Rizky Maulana Nugraha ' __date__ = '7/18/16' @@ -37,5 +38,7 @@ def ash_post_save(sender, instance, **kwargs): if instance.analysis_flag: for lang in ANALYSIS_LANGUAGES: generate_event_report.delay(instance, locale=lang) + push_hazard_to_geonode.delay(sender, instance) + except BaseException as e: LOGGER.exception(e) diff --git a/django_project/realtime/signals/earthquake.py b/django_project/realtime/signals/earthquake.py index aa5306be..46e18983 100644 --- a/django_project/realtime/signals/earthquake.py +++ b/django_project/realtime/signals/earthquake.py @@ -7,6 +7,7 @@ from realtime.app_settings import LOGGER_NAME, ANALYSIS_LANGUAGES from realtime.models.earthquake import Earthquake from realtime.tasks.earthquake import generate_event_report +from realtime.tasks.geonode import push_hazard_to_geonode __author__ = 'Rizky Maulana Nugraha ' __date__ = '7/18/16' @@ -31,5 +32,6 @@ def earthquake_post_save(sender, instance, **kwargs): for lang in ANALYSIS_LANGUAGES: generate_event_report.delay( instance, locale=lang) + push_hazard_to_geonode.delay(sender, instance) except BaseException: pass diff --git a/django_project/realtime/signals/flood.py b/django_project/realtime/signals/flood.py index 4f07570f..2aa95d70 100644 --- a/django_project/realtime/signals/flood.py +++ b/django_project/realtime/signals/flood.py @@ -7,6 +7,7 @@ from realtime.app_settings import LOGGER_NAME, ANALYSIS_LANGUAGES from realtime.models.flood import Flood from realtime.tasks.flood import generate_event_report +from realtime.tasks.geonode import push_hazard_to_geonode __author__ = 'Rizky Maulana Nugraha ' __date__ = '12/4/15' @@ -30,5 +31,7 @@ def flood_post_save( if instance.analysis_flag: for lang in ANALYSIS_LANGUAGES: generate_event_report.delay(instance, locale=lang) + if instance.analysis_flag: + push_hazard_to_geonode.delay(sender, instance) except Exception as e: LOGGER.exception(e) diff --git a/django_project/realtime/tasks/ash.py b/django_project/realtime/tasks/ash.py index 8b9cdbde..85632d39 100644 --- a/django_project/realtime/tasks/ash.py +++ b/django_project/realtime/tasks/ash.py @@ -21,8 +21,6 @@ from realtime.tasks.realtime.ash import process_ash from realtime.utils import substitute_layer_order, template_names, \ template_paths -from realtime.tasks.geonode import ( - upload_layer_to_geonode, handle_push_to_geonode) __author__ = 'Rizky Maulana Nugraha ' __date__ = '20/7/17' @@ -77,15 +75,6 @@ def generate_hazard_layer(ash_event): event_id=ash_event.id ).set(queue=handle_hazard_process.queue), - # # Push to GeoNode - upload_layer_to_geonode.s( - event_id=ash_event.id - ).set(queue=upload_layer_to_geonode.queue), - # - # # Handle push to geonode process - handle_push_to_geonode.s( - event_id=ash_event.id - ).set(queue=handle_push_to_geonode.queue) ) @app.task diff --git a/django_project/realtime/tasks/geonode.py b/django_project/realtime/tasks/geonode.py index 849cb59c..a80cb382 100644 --- a/django_project/realtime/tasks/geonode.py +++ b/django_project/realtime/tasks/geonode.py @@ -1,26 +1,24 @@ # coding=utf-8 """InaSAFE Django task related to GeoNode upload.""" from __future__ import absolute_import +import logging + +from celery import chain from core.celery_app import app from realtime.models.ash import Ash +from realtime.app_settings import LOGGER_NAME from realtime.tasks.headless.inasafe_wrapper import push_to_geonode +LOGGER = logging.getLogger(LOGGER_NAME) GEONODE_PUSH_SUCCESS = 0 -@app.task(queue='inasafe-django') -def upload_layer_to_geonode(process_result, event_id): - """Upload layer to geonode.""" - hazard = Ash.objects.get(id=event_id) - hazard_layer_uri = hazard.hazard_path - return push_to_geonode(hazard_layer_uri) @app.task(queue='inasafe-django') -def handle_push_to_geonode(push_result, event_id): +def handle_push_to_geonode(push_result, hazard_class, hazard_event_id): """Handle geonode push result.""" - # TODO(IS): make it generic for all hazard, somehow - hazard = Ash.objects.get(id=event_id) + hazard = hazard_class.objects.get(id=hazard_event_id) task_state = 'FAILURE' if not push_result: task_state = 'FAILURE' @@ -29,3 +27,32 @@ def handle_push_to_geonode(push_result, event_id): hazard.push_task_status = task_state hazard.push_task_result = push_result hazard.save() + + +@app.task(queue='inasafe-django') +def push_hazard_to_geonode(hazard_class, hazard_event): + """Upload layer to geonode and update the status of hazard.""" + LOGGER.info('Push layer to geonode.') + hazard_layer_uri = hazard_event.hazard_path + hazard_event_id = hazard_event.id + + tasks_chain = chain( + # Push to layer to geonode + push_to_geonode.s( + hazard_layer_uri + ).set(queue=push_to_geonode.queue), + + # Handle the push result + handle_push_to_geonode.s( + hazard_class, + hazard_event_id + ) + ) + + @app.task + def _handle_error(req, exc, traceback): + """Update task status as Failure.""" + hazard_event.push_task_status = 'FAILURE' + + async_result = tasks_chain.apply_async() + hazard_event.push_task_status = async_result.state diff --git a/django_project/realtime/tasks/test/test_geonode_task.py b/django_project/realtime/tasks/test/test_geonode_task.py new file mode 100644 index 00000000..ff4c56c8 --- /dev/null +++ b/django_project/realtime/tasks/test/test_geonode_task.py @@ -0,0 +1,20 @@ +# coding=utf-8 +import os + +from django.test import TestCase + +from realtime.tasks.geonode import push_hazard_to_geonode +from realtime.models.flood import Flood +from realtime.tests.model_factories import FloodFactory + + +class TestModelFlood(TestCase): + + @TestCase.skipTest('Under development') + def test_upload_flood_hazard(self): + flood = FloodFactory.create() + message = 'The flood object is instantiated successfully.' + self.assertIsNotNone(flood.id, message) + self.assertTrue(os.path.exists(flood.hazard_path)) + async_result = push_hazard_to_geonode.delay(Flood, flood) + _ = async_result.get() # noqa From fbf7a912ff7b3afa6af463feca61d41d2c58beca Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Thu, 7 Jun 2018 15:49:48 +0700 Subject: [PATCH 09/15] Trying to avoid passing python object in celery task. --- deployment/Makefile | 7 +++++++ django_project/realtime/signals/flood.py | 2 +- django_project/realtime/tasks/geonode.py | 17 ++++++++++++++--- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/deployment/Makefile b/deployment/Makefile index 9fbd6d49..6167711a 100644 --- a/deployment/Makefile +++ b/deployment/Makefile @@ -494,3 +494,10 @@ load-flood-test-data: @echo "Load flood test data" @echo "------------------------------------------------------------------" @docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py loadfloodtestdata + +superuser: + @echo + @echo "------------------------------------------------------------------" + @echo "Creating a superuser in production mode" + @echo "------------------------------------------------------------------" + @docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py createsuperuser diff --git a/django_project/realtime/signals/flood.py b/django_project/realtime/signals/flood.py index 2aa95d70..e8e28da5 100644 --- a/django_project/realtime/signals/flood.py +++ b/django_project/realtime/signals/flood.py @@ -32,6 +32,6 @@ def flood_post_save( for lang in ANALYSIS_LANGUAGES: generate_event_report.delay(instance, locale=lang) if instance.analysis_flag: - push_hazard_to_geonode.delay(sender, instance) + push_hazard_to_geonode.delay(instance) except Exception as e: LOGGER.exception(e) diff --git a/django_project/realtime/tasks/geonode.py b/django_project/realtime/tasks/geonode.py index a80cb382..f9cb2cbc 100644 --- a/django_project/realtime/tasks/geonode.py +++ b/django_project/realtime/tasks/geonode.py @@ -8,16 +8,25 @@ from core.celery_app import app from realtime.models.ash import Ash +from realtime.models.flood import Flood +from realtime.models.earthquake import Earthquake from realtime.app_settings import LOGGER_NAME from realtime.tasks.headless.inasafe_wrapper import push_to_geonode LOGGER = logging.getLogger(LOGGER_NAME) GEONODE_PUSH_SUCCESS = 0 +hazard_class_mapping = { + Ash.__name__: Ash, + Flood.__name__: Flood, + Earthquake.__name__: Earthquake +} + @app.task(queue='inasafe-django') -def handle_push_to_geonode(push_result, hazard_class, hazard_event_id): +def handle_push_to_geonode(push_result, hazard_class_name, hazard_event_id): """Handle geonode push result.""" + hazard_class = hazard_class_mapping.get(hazard_class_name) hazard = hazard_class.objects.get(id=hazard_event_id) task_state = 'FAILURE' if not push_result: @@ -30,11 +39,13 @@ def handle_push_to_geonode(push_result, hazard_class, hazard_event_id): @app.task(queue='inasafe-django') -def push_hazard_to_geonode(hazard_class, hazard_event): +def push_hazard_to_geonode(hazard_event): """Upload layer to geonode and update the status of hazard.""" LOGGER.info('Push layer to geonode.') hazard_layer_uri = hazard_event.hazard_path hazard_event_id = hazard_event.id + hazard_class = hazard_event.__class__ + hazard_class_name = hazard_class.__name__ tasks_chain = chain( # Push to layer to geonode @@ -44,7 +55,7 @@ def push_hazard_to_geonode(hazard_class, hazard_event): # Handle the push result handle_push_to_geonode.s( - hazard_class, + hazard_class_name, hazard_event_id ) ) From 064cc2c77d3017bc6ecfe8fbbb32458ff9a18549 Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Thu, 7 Jun 2018 16:00:37 +0700 Subject: [PATCH 10/15] Show geonode status in admin page. --- django_project/realtime/admin.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/django_project/realtime/admin.py b/django_project/realtime/admin.py index b7dd4a4d..30628563 100644 --- a/django_project/realtime/admin.py +++ b/django_project/realtime/admin.py @@ -85,8 +85,9 @@ class EarthquakeReportInline(StackedInline): class EarthquakeAdmin(LeafletGeoAdmin): """Admin Class for Earthquake Model.""" - list_display = ('shake_id', 'source_type', 'time', 'location_description', - 'magnitude', 'depth') + list_display = ( + 'shake_id', 'source_type', 'time', 'location_description', + 'magnitude', 'depth', 'push_task_status', 'push_task_result') list_filter = ('location_description', ) search_fields = ['shake_id', 'location_description'] inlines = [ @@ -125,8 +126,9 @@ class FloodReportInline(StackedInline): class FloodAdmin(ModelAdmin): """Admin Class for Flood Event.""" - list_display = ('event_id', 'data_source', 'time', - 'total_affected', 'boundary_flooded') + list_display = ( + 'event_id', 'data_source', 'time', 'total_affected', + 'boundary_flooded', 'push_task_status', 'push_task_result') inlines = [ ImpactInline, @@ -151,9 +153,10 @@ class AshReportInline(StackedInline): class AshAdmin(ModelAdmin): """Admin class for Ash model""" - list_display = ('volcano', 'alert_level', 'event_time', - 'event_time_zone_string', 'eruption_height', - 'forecast_duration') + list_display = ( + 'volcano', 'alert_level', 'event_time', + 'event_time_zone_string', 'eruption_height', + 'forecast_duration', 'push_task_status', 'push_task_result') inlines = [ ImpactInline, AshReportInline From 05d80a2ef54bc3aa434b58c687bce1b370d67b05 Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Thu, 7 Jun 2018 20:42:23 +0700 Subject: [PATCH 11/15] Add hacky way to make the celery task work for geonode push. --- django_project/realtime/tasks/geonode.py | 33 ++++++++++++++++-------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/django_project/realtime/tasks/geonode.py b/django_project/realtime/tasks/geonode.py index f9cb2cbc..3d32b1af 100644 --- a/django_project/realtime/tasks/geonode.py +++ b/django_project/realtime/tasks/geonode.py @@ -16,32 +16,43 @@ LOGGER = logging.getLogger(LOGGER_NAME) GEONODE_PUSH_SUCCESS = 0 -hazard_class_mapping = { - Ash.__name__: Ash, - Flood.__name__: Flood, - Earthquake.__name__: Earthquake -} - @app.task(queue='inasafe-django') def handle_push_to_geonode(push_result, hazard_class_name, hazard_event_id): """Handle geonode push result.""" + hazard_class_mapping = { + Ash.__name__: Ash, + Flood.__name__: Flood, + Earthquake.__name__: Earthquake + } + + # Hacky thing to get proper class + if hazard_class_name not in hazard_class_mapping.keys(): + for key in hazard_class_mapping.keys(): + if key.lower() in hazard_class_name.lower(): + hazard_class_name = key + break hazard_class = hazard_class_mapping.get(hazard_class_name) - hazard = hazard_class.objects.get(id=hazard_event_id) task_state = 'FAILURE' if not push_result: task_state = 'FAILURE' elif push_result['status'] == GEONODE_PUSH_SUCCESS: task_state = 'SUCCESS' - hazard.push_task_status = task_state - hazard.push_task_result = push_result - hazard.save() + + # Use update so it doesn't trigger save signal + hazard_class.objects.filter(id=hazard_event_id).update( + push_task_status=task_state, + push_task_result=push_result + ) @app.task(queue='inasafe-django') def push_hazard_to_geonode(hazard_event): """Upload layer to geonode and update the status of hazard.""" LOGGER.info('Push layer to geonode.') + # Skip if it's already running + if hazard_event.push_task_status: + return hazard_layer_uri = hazard_event.hazard_path hazard_event_id = hazard_event.id hazard_class = hazard_event.__class__ @@ -57,7 +68,7 @@ def push_hazard_to_geonode(hazard_event): handle_push_to_geonode.s( hazard_class_name, hazard_event_id - ) + ).set(queue=handle_push_to_geonode.queue) ) @app.task From b63d0bc60ba6609722ef052d8e2e0ed9911bf18c Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Thu, 7 Jun 2018 20:59:36 +0700 Subject: [PATCH 12/15] Handling disabled geonode push. --- .../templates/docker-compose.override.yml.j2 | 1 + django_project/realtime/app_settings.py | 2 ++ django_project/realtime/tasks/geonode.py | 12 ++++++++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 b/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 index 596f7f50..fc03c6f3 100644 --- a/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 +++ b/deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2 @@ -76,6 +76,7 @@ services: - TASK_ALWAYS_EAGER={{ inasafe_django.task_always_eager }} - INASAFE_OUTPUT_DIR={{ inasafe_headless.working_dir }}/outputs - ON_TRAVIS={{ on_travis }} + - REALTIME_GEONODE_ENABLE={{ geonode.enable }} links: - smtp:smtp - db:db diff --git a/django_project/realtime/app_settings.py b/django_project/realtime/app_settings.py index 100cfbe5..5265b4de 100644 --- a/django_project/realtime/app_settings.py +++ b/django_project/realtime/app_settings.py @@ -19,6 +19,8 @@ LOGGER_NAME = 'InaSAFE Realtime REST Server' ON_TRAVIS = ast.literal_eval(os.environ.get('ON_TRAVIS', 'False')) +REALTIME_GEONODE_ENABLE = ast.literal_eval( + os.environ.get('REALTIME_GEONODE_ENABLE', 'False')) # PROJECT_NAME: The project name for this apps e.g InaSAFE default_project_name = 'InaSAFE Realtime' diff --git a/django_project/realtime/tasks/geonode.py b/django_project/realtime/tasks/geonode.py index 3d32b1af..8f755b30 100644 --- a/django_project/realtime/tasks/geonode.py +++ b/django_project/realtime/tasks/geonode.py @@ -2,6 +2,7 @@ """InaSAFE Django task related to GeoNode upload.""" from __future__ import absolute_import import logging +import os from celery import chain @@ -10,7 +11,7 @@ from realtime.models.ash import Ash from realtime.models.flood import Flood from realtime.models.earthquake import Earthquake -from realtime.app_settings import LOGGER_NAME +from realtime.app_settings import LOGGER_NAME, REALTIME_GEONODE_ENABLE from realtime.tasks.headless.inasafe_wrapper import push_to_geonode LOGGER = logging.getLogger(LOGGER_NAME) @@ -49,8 +50,15 @@ def handle_push_to_geonode(push_result, hazard_class_name, hazard_event_id): @app.task(queue='inasafe-django') def push_hazard_to_geonode(hazard_event): """Upload layer to geonode and update the status of hazard.""" + # If geonode push is disabled, skip the task + if not REALTIME_GEONODE_ENABLE: + hazard_event.__class__.objects.filter(id=hazard_event.id).update( + push_task_status='DISABLED', + push_task_result='GeoNode push is disabled in the setting.' + ) + return LOGGER.info('Push layer to geonode.') - # Skip if it's already running + # Skip if it's already running. if hazard_event.push_task_status: return hazard_layer_uri = hazard_event.hazard_path From 03f431dddc070953903f077cf43459eea260cb62 Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Fri, 8 Jun 2018 01:10:41 +0700 Subject: [PATCH 13/15] Fix failed travis. --- django_project/realtime/tasks/geonode.py | 1 - django_project/realtime/tasks/test/test_geonode_task.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/django_project/realtime/tasks/geonode.py b/django_project/realtime/tasks/geonode.py index 8f755b30..335c6bcb 100644 --- a/django_project/realtime/tasks/geonode.py +++ b/django_project/realtime/tasks/geonode.py @@ -2,7 +2,6 @@ """InaSAFE Django task related to GeoNode upload.""" from __future__ import absolute_import import logging -import os from celery import chain diff --git a/django_project/realtime/tasks/test/test_geonode_task.py b/django_project/realtime/tasks/test/test_geonode_task.py index ff4c56c8..e86c98c8 100644 --- a/django_project/realtime/tasks/test/test_geonode_task.py +++ b/django_project/realtime/tasks/test/test_geonode_task.py @@ -1,6 +1,6 @@ # coding=utf-8 import os - +from unittest import skip from django.test import TestCase from realtime.tasks.geonode import push_hazard_to_geonode @@ -10,7 +10,7 @@ class TestModelFlood(TestCase): - @TestCase.skipTest('Under development') + @skip('Under development') def test_upload_flood_hazard(self): flood = FloodFactory.create() message = 'The flood object is instantiated successfully.' From 686c42c6b9bfb6960501112d4a690d47dcae271b Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Fri, 8 Jun 2018 06:00:11 +0700 Subject: [PATCH 14/15] Add geonode variable in PyCharm run environment. --- .../roles/pycharm/templates/run-manager-environment.xml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 b/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 index 3824a661..5dc4d50a 100644 --- a/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 +++ b/deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2 @@ -8,6 +8,7 @@ + From 3638b49d793bdfb3be478ee16b492a2dad7517e3 Mon Sep 17 00:00:00 2001 From: Ismail Sunni Date: Fri, 8 Jun 2018 07:03:09 +0700 Subject: [PATCH 15/15] Add unit test for upload geojson. --- .../realtime/tasks/test/test_headless_task.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/django_project/realtime/tasks/test/test_headless_task.py b/django_project/realtime/tasks/test/test_headless_task.py index 88ab2635..0a7a32ec 100644 --- a/django_project/realtime/tasks/test/test_headless_task.py +++ b/django_project/realtime/tasks/test/test_headless_task.py @@ -13,7 +13,8 @@ ASH_LAYER_ORDER, FLOOD_EXPOSURE, FLOOD_REPORT_TEMPLATE_EN, - FLOOD_LAYER_ORDER + FLOOD_LAYER_ORDER, + REALTIME_GEONODE_ENABLE, ) from realtime.tasks.headless.celery_app import app as headless_app from realtime.tasks.headless.inasafe_wrapper import ( @@ -457,9 +458,16 @@ def test_flood_analysis_real_exposure(self): self.assertNotIn('inasafe-map-report-portrait', product_keys) self.assertNotIn('inasafe-map-report-landscape', product_keys) - @unittest.skipIf(os.environ.get('ON_TRAVIS', False), 'No geonode instance') + @unittest.skipIf(not REALTIME_GEONODE_ENABLE, 'GeoNode push is disabled.') def test_push_tif_to_geonode(self): """Test push tif layer to geonode functionality.""" async_result = push_to_geonode.delay(shakemap_layer_uri) result = async_result.get() self.assertEqual(result['status'], 0, result['message']) + + @unittest.skipIf(not REALTIME_GEONODE_ENABLE, 'GeoNode push is disabled.') + def test_push_geojson_to_geonode(self): + """Test push geojson layer to geonode functionality.""" + async_result = push_to_geonode.delay(flood_layer_uri) + result = async_result.get() + self.assertEqual(result['status'], 0, result['message'])