diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b268158ba1..e19d3d2bdf2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,11 +93,6 @@ jobs: command: docker logs django4geonode --tail 500 when: on_fail - - run: - name: Invoke logs (debug) - command: docker exec -it django4geonode sh -c 'tail -1000 /usr/src/geonode/invoke.log' - when: on_fail - - when: condition: <> steps: @@ -126,44 +121,24 @@ workflows: jobs: - build: name: geonode_test_suite_smoke - load_docker_cache: true - save_docker_cache: true + load_docker_cache: false + save_docker_cache: false test_suite: ./test.sh geonode.tests.smoke geonode.tests.test_message_notifications geonode.tests.test_rest_api geonode.tests.test_search geonode.tests.test_utils - build: name: geonode_test_suite - load_docker_cache: true + load_docker_cache: false save_docker_cache: false test_suite: ./test.sh $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS]))") geonode.catalogue.backends.tests geonode.thumbs.tests - requires: - - geonode_test_suite_smoke - build: name: geonode_test_rest_apis - load_docker_cache: true + load_docker_cache: false save_docker_cache: false - test_suite: ./test.sh geonode.base.api.tests geonode.layers.api.tests geonode.maps.api.tests geonode.documents.api.tests geonode.geoapps.api.tests - requires: - - geonode_test_suite_smoke + test_suite: ./test.sh geonode.base.api.tests geonode.layers.api.tests geonode.maps.api.tests geonode.documents.api.tests geonode.geoapps.api.tests geonode.upload.api.tests - build: name: geonode_test_integration_csw - load_docker_cache: true + load_docker_cache: false save_docker_cache: false test_suite: ./test_csw.sh - requires: - - geonode_test_suite_smoke - - build: - name: geonode_test_integration_monitoring - load_docker_cache: true - save_docker_cache: false - test_suite: ./test.sh geonode.tests.smoke geonode.monitoring.tests.integration - requires: - - geonode_test_suite_smoke - - build: - name: geonode_test_integration_upload - load_docker_cache: true - save_docker_cache: false - test_suite: ./test.sh geonode.upload.api.tests - requires: - - geonode_test_suite_smoke # TODO # - build: @@ -184,7 +159,7 @@ workflows: # name: tests_geoserver_integration # load_docker_cache: true # save_docker_cache: false - # test_suite: 'geonode.geoserver.tests.integration' + # test_suite: 'geonode.geoserver.tests.integration geonode.monitoring.tests.integration' # requires: # - geonode_test_suite_smoke diff --git a/.env b/.env index 8dd154fdc5f..7fdef719726 100644 --- a/.env +++ b/.env @@ -8,14 +8,15 @@ BACKUPS_VOLUME_DRIVER=local C_FORCE_ROOT=1 FORCE_REINIT=false +INVOKE_LOG_STDOUT=true # LANGUAGE_CODE=pt # LANGUAGES=(('en','English'),('pt','Portuguese')) DJANGO_SETTINGS_MODULE=geonode.settings GEONODE_INSTANCE_NAME=geonode -GEONODE_LB_HOST_IP -GEONODE_LB_PORT +GEONODE_LB_HOST_IP= +GEONODE_LB_PORT= # ################# # backend @@ -119,7 +120,7 @@ DEFAULT_FROM_EMAIL='GeoNode ' # Session/Access Control LOCKDOWN_GEONODE=False CORS_ORIGIN_ALLOW_ALL=True -X_FRAME_OPTIONS=ALLOW-FROM ALL +X_FRAME_OPTIONS="ALLOW-FROM ALL" SESSION_EXPIRED_CONTROL_ENABLED=True DEFAULT_ANONYMOUS_VIEW_PERMISSION=True DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True diff --git a/.env_test b/.env_test index 00773a457cb..3a7a9ca2979 100644 --- a/.env_test +++ b/.env_test @@ -7,14 +7,20 @@ DOCKER_API_VERSION="1.24" BACKUPS_VOLUME_DRIVER=local C_FORCE_ROOT=1 +FORCE_REINIT=false +INVOKE_LOG_STDOUT=true -DEBUG=False +# LANGUAGE_CODE=pt +# LANGUAGES=(('en','English'),('pt','Portuguese')) DJANGO_SETTINGS_MODULE=geonode.settings GEONODE_INSTANCE_NAME=geonode GEONODE_LB_HOST_IP= GEONODE_LB_PORT= +# ################# +# backend +# ################# POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres GEONODE_DATABASE=geonode @@ -23,6 +29,8 @@ GEONODE_GEODATABASE=geonode_data GEONODE_GEODATABASE_PASSWORD=geonode_data GEONODE_DATABASE_SCHEMA=public GEONODE_GEODATABASE_SCHEMA=public +DATABASE_HOST=db +DATABASE_PORT=5432 DATABASE_URL=postgis://geonode:geonode@db:5432/geonode GEODATABASE_URL=postgis://geonode_data:geonode_data@db:5432/geonode_data GEONODE_DB_CONN_MAX_AGE=0 @@ -33,12 +41,9 @@ ASYNC_SIGNALS=True SITEURL=http://localhost:8001/ -STATIC_ROOT=/mnt/volumes/statics/static/ -MEDIA_ROOT=/mnt/volumes/statics/uploaded/ -GEOIP_PATH=/mnt/volumes/statics/geoip.db - ALLOWED_HOSTS="['django', '*']" +# Data Uploader DEFAULT_BACKEND_UPLOADER=geonode.importer TIME_ENABLED=True MOSAIC_ENABLED=False @@ -47,20 +52,10 @@ HAYSTACK_ENGINE_URL=http://elasticsearch:9200/ HAYSTACK_ENGINE_INDEX_NAME=haystack HAYSTACK_SEARCH_RESULTS_PER_PAGE=200 -CACHE_BUSTING_STATIC_ENABLED=False -CACHE_BUSTING_MEDIA_ENABLED=False - -MEMCACHED_ENABLED=False -MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyMemcacheCache -MEMCACHED_LOCATION=127.0.0.1:11211 -MEMCACHED_LOCK_EXPIRE=3600 -MEMCACHED_LOCK_TIMEOUT=10 - -MAX_DOCUMENT_SIZE=2 -CLIENT_RESULTS_LIMIT=5 -API_LIMIT_PER_PAGE=1000 - +# ################# +# nginx # HTTPD Server +# ################# GEONODE_LB_HOST_IP=localhost GEONODE_LB_PORT=80 @@ -83,7 +78,9 @@ LETSENCRYPT_MODE=disabled RESOLVER=127.0.0.11 -# GIS Server +# ################# +# geoserver +# ################# GEOSERVER_WEB_UI_LOCATION=http://localhost:8001/geoserver/ GEOSERVER_PUBLIC_LOCATION=http://localhost:8001/geoserver/ GEOSERVER_LOCATION=http://geoserver:8080/geoserver/ @@ -96,29 +93,10 @@ OGC_REQUEST_BACKOFF_FACTOR=0.3 OGC_REQUEST_POOL_MAXSIZE=10 OGC_REQUEST_POOL_CONNECTIONS=10 -# GIS Client -GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore -MAPBOX_ACCESS_TOKEN= -BING_API_KEY= -GOOGLE_API_KEY= - -# Monitoring -MONITORING_ENABLED=True -MONITORING_DATA_TTL=365 -USER_ANALYTICS_ENABLED=True -USER_ANALYTICS_GZIP=True -CENTRALIZED_DASHBOARD_ENABLED=False -MONITORING_SERVICE_NAME=local-geonode -MONITORING_HOST_NAME=geonode - -# Other Options/Contribs -MODIFY_TOPICCATEGORY=True -AVATAR_GRAVATAR_SSL=True -AVATAR_DEFAULT_URL=/geonode/img/avatar.png - -EXIF_ENABLED=True -CREATE_LAYER=True -FAVORITE_ENABLED=True +# Java Options & Memory +ENABLE_JSONP=true +outFormat=text/javascript +GEOSERVER_JAVA_OPTS="-Djava.awt.headless=true -Xms2G -Xmx4G -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://geoserver:8080/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine" # ################# # Security @@ -166,3 +144,51 @@ OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZK # GeoNode APIs API_LOCKDOWN=False TASTYPIE_APIKEY= + +# ################# +# Production and +# Monitoring +# ################# +DEBUG=False + +SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' + +STATIC_ROOT=/mnt/volumes/statics/static/ +MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +GEOIP_PATH=/mnt/volumes/statics/geoip.db + +CACHE_BUSTING_STATIC_ENABLED=False +CACHE_BUSTING_MEDIA_ENABLED=False + +MEMCACHED_ENABLED=False +MEMCACHED_BACKEND=django.core.cache.backends.memcached.MemcachedCache +MEMCACHED_LOCATION=127.0.0.1:11211 +MEMCACHED_LOCK_EXPIRE=3600 +MEMCACHED_LOCK_TIMEOUT=10 + +MAX_DOCUMENT_SIZE=2 +CLIENT_RESULTS_LIMIT=5 +API_LIMIT_PER_PAGE=1000 + +# GIS Client +GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore +MAPBOX_ACCESS_TOKEN= +BING_API_KEY= +GOOGLE_API_KEY= + +# Monitoring +MONITORING_ENABLED=True +MONITORING_DATA_TTL=365 +USER_ANALYTICS_ENABLED=True +USER_ANALYTICS_GZIP=True +CENTRALIZED_DASHBOARD_ENABLED=False +MONITORING_SERVICE_NAME=local-geonode +MONITORING_HOST_NAME=geonode + +# Other Options/Contribs +MODIFY_TOPICCATEGORY=True +AVATAR_GRAVATAR_SSL=True +EXIF_ENABLED=True +CREATE_LAYER=True +FAVORITE_ENABLED=True + diff --git a/Dockerfile b/Dockerfile index 6f3a65bb349..383b773e013 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,11 +64,11 @@ RUN chmod +x /usr/bin/celery-commands COPY celery-cmd /usr/bin/celery-cmd RUN chmod +x /usr/bin/celery-cmd -# Install "geonode-contribs" apps -RUN cd /usr/src; git clone https://github.com/GeoNode/geonode-contribs.git -b master -# Install logstash and centralized dashboard dependencies -RUN cd /usr/src/geonode-contribs/geonode-logstash; pip install --upgrade -e . \ - cd /usr/src/geonode-contribs/ldap; pip install --upgrade -e . +# # Install "geonode-contribs" apps +# RUN cd /usr/src; git clone https://github.com/GeoNode/geonode-contribs.git -b master +# # Install logstash and centralized dashboard dependencies +# RUN cd /usr/src/geonode-contribs/geonode-logstash; pip install --upgrade -e . \ +# cd /usr/src/geonode-contribs/ldap; pip install --upgrade -e . RUN pip install --upgrade --no-cache-dir --src /usr/src -r requirements.txt RUN pip install --upgrade -e . diff --git a/entrypoint.sh b/entrypoint.sh index f3ce10a058e..bec93ec5888 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,6 +3,17 @@ # Exit script in case of error set -e +INVOKE_LOG_STDOUT=${INVOKE_LOG_STDOUT:-FALSE} +invoke () { + if [ $INVOKE_LOG_STDOUT = 'true' ] || [ $INVOKE_LOG_STDOUT = 'True' ] + then + /usr/local/bin/invoke $@ + else + /usr/local/bin/invoke $@ > /usr/src/geonode/invoke.log 2>&1 + fi + echo "$@ tasks done" +} + # Start cron && memcached services service cron restart service memcached restart @@ -12,7 +23,7 @@ echo "-----------------------------------------------------" echo "STARTING DJANGO ENTRYPOINT $(date)" echo "-----------------------------------------------------" -/usr/local/bin/invoke update > /usr/src/geonode/invoke.log 2>&1 +invoke update source $HOME/.bashrc source $HOME/.override_env @@ -30,8 +41,7 @@ echo MONITORING_HOST_NAME=$MONITORING_HOST_NAME echo MONITORING_SERVICE_NAME=$MONITORING_SERVICE_NAME echo MONITORING_DATA_TTL=$MONITORING_DATA_TTL -/usr/local/bin/invoke waitfordbs > /usr/src/geonode/invoke.log 2>&1 -echo "waitfordbs task done" +invoke waitfordbs cmd="$@" @@ -40,13 +50,9 @@ echo DOCKER_ENV=$DOCKER_ENV if [ -z ${DOCKER_ENV} ] || [ ${DOCKER_ENV} = "development" ] then - echo "running migrations" - /usr/local/bin/invoke migrations > /usr/src/geonode/invoke.log 2>&1 - echo "migrations task done" - /usr/local/bin/invoke prepare > /usr/src/geonode/invoke.log 2>&1 - echo "prepare task done" - /usr/local/bin/invoke fixtures > /usr/src/geonode/invoke.log 2>&1 - echo "fixture task done" + invoke migrations + invoke prepare + invoke fixtures if [ ${IS_CELERY} = "true" ] || [ ${IS_CELERY} = "True" ] then @@ -55,11 +61,8 @@ then else - echo "install requirements for development" - /usr/local/bin/invoke devrequirements > /usr/src/geonode/invoke.log 2>&1 - echo "refresh static data" - /usr/local/bin/invoke statics > /usr/src/geonode/invoke.log 2>&1 - echo "static data refreshed" + invoke devrequirements + invoke statics echo "Executing standard Django server $cmd for Development" @@ -71,32 +74,20 @@ else echo "Executing Celery server $cmd for Production" else - echo "running migrations" - /usr/local/bin/invoke migrations > /usr/src/geonode/invoke.log 2>&1 - echo "migrations task done" - /usr/local/bin/invoke prepare > /usr/src/geonode/invoke.log 2>&1 - echo "prepare task done" + invoke migrations + invoke prepare if [ ${FORCE_REINIT} = "true" ] || [ ${FORCE_REINIT} = "True" ] || [ ! -e "/mnt/volumes/statics/geonode_init.lock" ]; then - /usr/local/bin/invoke updategeoip > /usr/src/geonode/invoke.log 2>&1 - echo "updategeoip task done" - /usr/local/bin/invoke fixtures > /usr/src/geonode/invoke.log 2>&1 - echo "fixture task done" - /usr/local/bin/invoke monitoringfixture > /usr/src/geonode/invoke.log 2>&1 - echo "monitoringfixture task done" - /usr/local/bin/invoke initialized > /usr/src/geonode/invoke.log 2>&1 - echo "initialized" + invoke updategeoip + invoke fixtures + invoke monitoringfixture + invoke initialized fi - echo "refresh static data" - /usr/local/bin/invoke statics > /usr/src/geonode/invoke.log 2>&1 - echo "static data refreshed" - /usr/local/bin/invoke waitforgeoserver > /usr/src/geonode/invoke.log 2>&1 - echo "waitforgeoserver task done" - /usr/local/bin/invoke geoserverfixture > /usr/src/geonode/invoke.log 2>&1 - echo "geoserverfixture task done" - /usr/local/bin/invoke updateadmin > /usr/src/geonode/invoke.log 2>&1 - echo "updateadmin task done" + invoke statics + invoke waitforgeoserver + invoke geoserverfixture + invoke updateadmin echo "Executing UWSGI server $cmd for Production" fi diff --git a/geonode/__init__.py b/geonode/__init__.py index 3a0894af509..7ac4383a303 100644 --- a/geonode/__init__.py +++ b/geonode/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/api/__init__.py b/geonode/api/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/api/__init__.py +++ b/geonode/api/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/api/api.py b/geonode/api/api.py index 90f500bb50c..fc66e480554 100644 --- a/geonode/api/api.py +++ b/geonode/api/api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -117,8 +116,8 @@ def get_resources_counts(self, options): else: counts = list(resources.values(options['count_type']).annotate(count=Count(options['count_type']))) - return dict( - [(c[options['count_type']], c['count']) for c in counts if c and c['count'] and options['count_type']]) + return { + c[options['count_type']]: c['count'] for c in counts if c and c['count'] and options['count_type']} def to_json(self, data, options=None): options = options or {} @@ -146,7 +145,7 @@ def build_filters(self, filters=None, ignore_bad_filters=False): self.type_filter = None self.title_filter = None - orm_filters = super(TypeFilteredResource, self).build_filters(filters) + orm_filters = super().build_filters(filters) if 'type' in filters and filters['type'] in FILTER_TYPES.keys(): self.type_filter = FILTER_TYPES[filters['type']] @@ -164,7 +163,7 @@ def serialize(self, request, data, format, options=None): options['type_filter'] = getattr(self, 'type_filter', None) options['user'] = request.user - return super(TypeFilteredResource, self).serialize(request, data, format, options) + return super().serialize(request, data, format, options) class TagResource(TypeFilteredResource): @@ -175,7 +174,7 @@ def serialize(self, request, data, format, options=None): options = {} options['count_type'] = 'keywords' - return super(TagResource, self).serialize(request, data, format, options) + return super().serialize(request, data, format, options) class Meta: queryset = HierarchicalKeyword.objects.all().order_by('name') @@ -198,7 +197,7 @@ def build_filters(self, filters={}, ignore_bad_filters=False): """adds filtering by current language""" _filters = filters.copy() id = _filters.pop('id', None) - orm_filters = super(ThesaurusKeywordResource, self).build_filters(_filters) + orm_filters = super().build_filters(_filters) if id is not None: orm_filters['id__in'] = id @@ -211,7 +210,7 @@ def build_filters(self, filters={}, ignore_bad_filters=False): def serialize(self, request, data, format, options={}): options['count_type'] = 'tkeywords__id' - return super(ThesaurusKeywordResource, self).serialize(request, data, format, options) + return super().serialize(request, data, format, options) def dehydrate_id(self, bundle): return bundle.obj.id @@ -259,7 +258,7 @@ def serialize(self, request, data, format, options=None): options = {} options['count_type'] = 'regions' - return super(RegionResource, self).serialize(request, data, format, options) + return super().serialize(request, data, format, options) class Meta: queryset = Region.objects.all().order_by('name') @@ -299,7 +298,7 @@ def serialize(self, request, data, format, options=None): options = {} options['count_type'] = 'category' - return super(TopicCategoryResource, self).serialize(request, data, format, options) + return super().serialize(request, data, format, options) class Meta: queryset = TopicCategory.objects.all() @@ -327,9 +326,7 @@ class Meta: authorization = ApiLockdownAuthorization() def apply_filters(self, request, applicable_filters): - filtered = super( - GroupCategoryResource, - self).apply_filters( + filtered = super().apply_filters( request, applicable_filters) return filtered @@ -445,7 +442,7 @@ def get_object_list(self, request): """ - qs = super(GroupResource, self).get_object_list(request) + qs = super().get_object_list(request) return qs.exclude(name="anonymous") @@ -465,7 +462,7 @@ def build_filters(self, filters=None, ignore_bad_filters=False): if filters is None: filters = {} - orm_filters = super(ProfileResource, self).build_filters(filters) + orm_filters = super().build_filters(filters) if 'group' in filters: orm_filters['group'] = filters['group'] @@ -481,9 +478,7 @@ def apply_filters(self, request, applicable_filters): group = applicable_filters.pop('group', None) name = applicable_filters.pop('name__icontains', None) - semi_filtered = super( - ProfileResource, - self).apply_filters( + semi_filtered = super().apply_filters( request, applicable_filters) @@ -562,7 +557,7 @@ def dehydrate(self, bundle): def prepend_urls(self): if settings.HAYSTACK_SEARCH: return [ - url(r"^(?P%s)/search%s$" % ( + url(r"^(?P{})/search{}$".format( self._meta.resource_name, trailing_slash() ), self.wrap_view('get_search'), name="api_get_search"), @@ -575,7 +570,7 @@ def serialize(self, request, data, format, options=None): options = {} options['count_type'] = 'owner' - return super(ProfileResource, self).serialize(request, data, format, options) + return super().serialize(request, data, format, options) class Meta: queryset = get_user_model().objects.exclude(Q(username='AnonymousUser') | Q(is_active=False)) @@ -620,7 +615,7 @@ def serialize(self, request, data, format, options=None): options = {} options['count_type'] = 'owner' - return super(OwnersResource, self).serialize(request, data, format, options) + return super().serialize(request, data, format, options) class Meta: queryset = get_user_model().objects.exclude(username='AnonymousUser') @@ -674,7 +669,7 @@ class Meta: def build_filters(self, filters=None, **kwargs): """Apply custom filters for layer.""" - filters = super(GeoserverStyleResource, self).build_filters( + filters = super().build_filters( filters, **kwargs) # Convert layer__ filters into layer_styles__layer__ updated_filters = {} diff --git a/geonode/api/authentication.py b/geonode/api/authentication.py index cc0815b8fae..f8f971f9b21 100644 --- a/geonode/api/authentication.py +++ b/geonode/api/authentication.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2019 OSGeo diff --git a/geonode/api/authorization.py b/geonode/api/authorization.py index d4e3d886540..702042e915a 100644 --- a/geonode/api/authorization.py +++ b/geonode/api/authorization.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -168,7 +167,7 @@ def read_list(self, object_list, bundle): class GroupAuthorization(ApiLockdownAuthorization): def read_list(self, object_list, bundle): - groups = super(GroupAuthorization, self).read_list(object_list, bundle) + groups = super().read_list(object_list, bundle) user = bundle.request.user if groups: if not user.is_authenticated or user.is_anonymous: @@ -181,7 +180,7 @@ def read_list(self, object_list, bundle): class GroupProfileAuthorization(ApiLockdownAuthorization): def read_list(self, object_list, bundle): - groups = super(GroupProfileAuthorization, self).read_list(object_list, bundle) + groups = super().read_list(object_list, bundle) user = bundle.request.user if groups: if not user.is_authenticated or user.is_anonymous: diff --git a/geonode/api/paginator.py b/geonode/api/paginator.py index 6de5be8ff65..403627d5644 100644 --- a/geonode/api/paginator.py +++ b/geonode/api/paginator.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo diff --git a/geonode/api/resourcebase_api.py b/geonode/api/resourcebase_api.py index 665acaf3d95..e397e18b4b8 100644 --- a/geonode/api/resourcebase_api.py +++ b/geonode/api/resourcebase_api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -17,6 +16,7 @@ # along with this program. If not, see . # ######################################################################### +from geonode.base.enumerations import LAYER_TYPES import re import logging @@ -72,14 +72,6 @@ logger = logging.getLogger(__name__) -LAYER_SUBTYPES = { - 'vector': 'dataStore', - 'raster': 'coverageStore', - 'remote': 'remoteStore', - 'vector_time': 'vectorTimeSeries', -} -FILTER_TYPES.update(LAYER_SUBTYPES) - class CommonMetaApi: authorization = GeoNodeAuthorization() @@ -162,9 +154,9 @@ class CommonModelApi(ModelResource): def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): if filters is None: filters = {} - orm_filters = super(CommonModelApi, self).build_filters( + orm_filters = super().build_filters( filters=filters, ignore_bad_filters=ignore_bad_filters, **kwargs) - if 'type__in' in filters and filters['type__in'] in FILTER_TYPES.keys(): + if 'type__in' in filters and (filters['type__in'] in FILTER_TYPES.keys() or filters['type__in'] in LAYER_TYPES): orm_filters.update({'type': filters.getlist('type__in')}) if 'app_type__in' in filters: orm_filters.update({'polymorphic_ctype__model': filters['app_type__in'].lower()}) @@ -193,32 +185,30 @@ def apply_filters(self, request, applicable_filters): filters |= Q(f) semi_filtered = self.get_object_list(request).filter(filters) else: - semi_filtered = super( - CommonModelApi, - self).apply_filters( + semi_filtered = super().apply_filters( request, applicable_filters) filtered = None if types: for the_type in types: - if the_type in LAYER_SUBTYPES.keys(): + if the_type in LAYER_TYPES: super_type = the_type if 'vector_time' == the_type: super_type = 'vector' if filtered: if 'time' in the_type: filtered = filtered | semi_filtered.filter( - Layer___storeType=LAYER_SUBTYPES[super_type]).exclude(Layer___has_time=False) + Layer___storetype=super_type).exclude(Layer___has_time=False) else: filtered = filtered | semi_filtered.filter( - Layer___storeType=LAYER_SUBTYPES[super_type]) + Layer___storetype=super_type) else: if 'time' in the_type: filtered = semi_filtered.filter( - Layer___storeType=LAYER_SUBTYPES[super_type]).exclude(Layer___has_time=False) + Layer___storetype=super_type).exclude(Layer___has_time=False) else: filtered = semi_filtered.filter( - Layer___storeType=LAYER_SUBTYPES[super_type]) + Layer___storetype=super_type) else: _type_filter = FILTER_TYPES[the_type].__name__.lower() if filtered: @@ -310,7 +300,7 @@ def build_haystack_filters(self, parameters): if type in {"map", "layer", "document", "user"}: # Type is one of our Major Types (not a sub type) types.append(type) - elif type in LAYER_SUBTYPES.keys(): + elif type in LAYER_TYPES: subtypes.append(type) if 'vector' in subtypes and 'vector_time' not in subtypes: @@ -644,7 +634,7 @@ def create_response( def prepend_urls(self): if settings.HAYSTACK_SEARCH: return [ - url(r"^(?P%s)/search%s$" % ( + url(r"^(?P{})/search{}$".format( self._meta.resource_name, trailing_slash() ), self.wrap_view('get_search'), name="api_get_search"), @@ -703,7 +693,7 @@ class LayerResource(CommonModelApi): def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): _filters = filters.copy() metadata_only = _filters.pop('metadata_only', False) - orm_filters = super(LayerResource, self).build_filters(_filters) + orm_filters = super().build_filters(_filters) orm_filters['metadata_only'] = False if not metadata_only else metadata_only[0] return orm_filters @@ -752,9 +742,9 @@ def format_objects(self, objects): # Probe Remote Services formatted_obj['store_type'] = 'dataset' formatted_obj['online'] = True - if hasattr(obj, 'storeType'): - formatted_obj['store_type'] = obj.storeType - if obj.storeType == 'remoteStore' and hasattr(obj, 'remote_service'): + if hasattr(obj, 'storetype'): + formatted_obj['store_type'] = obj.storetype + if obj.storetype in ['tileStore', 'remote'] and hasattr(obj, 'remote_service'): if obj.remote_service: formatted_obj['online'] = (obj.remote_service.probe == 200) else: @@ -861,7 +851,7 @@ class MapResource(CommonModelApi): def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): _filters = filters.copy() metadata_only = _filters.pop('metadata_only', False) - orm_filters = super(MapResource, self).build_filters(_filters) + orm_filters = super().build_filters(_filters) orm_filters['metadata_only'] = False if not metadata_only else metadata_only[0] return orm_filters @@ -1009,7 +999,7 @@ class DocumentResource(CommonModelApi): def build_filters(self, filters=None, ignore_bad_filters=False, **kwargs): _filters = filters.copy() metadata_only = _filters.pop('metadata_only', False) - orm_filters = super(DocumentResource, self).build_filters(_filters) + orm_filters = super().build_filters(_filters) orm_filters['metadata_only'] = False if not metadata_only else metadata_only[0] return orm_filters @@ -1061,7 +1051,7 @@ def format_objects(self, objects): class Meta(CommonMetaApi): paginator_class = CrossSiteXHRPaginator filtering = CommonMetaApi.filtering - filtering.update({'doc_type': ALL}) + filtering.update({'storetype': ALL}) queryset = Document.objects.distinct().order_by('-date') resource_name = 'documents' authentication = MultiAuthentication(SessionAuthentication(), diff --git a/geonode/api/tests.py b/geonode/api/tests.py index 14c0328a9fd..00a333694a3 100644 --- a/geonode/api/tests.py +++ b/geonode/api/tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -17,8 +16,6 @@ # along with this program. If not, see . # ######################################################################### -from geonode.maps.models import Map -from geonode.documents.models import Document from unittest.mock import patch from django.conf import settings @@ -33,27 +30,37 @@ from guardian.shortcuts import get_anonymous_user from geonode import geoserver +from geonode.maps.models import Map from geonode.layers.models import Layer +from geonode.documents.models import Document from geonode.utils import check_ogc_backend from geonode.decorators import on_ogc_backend from geonode.groups.models import GroupProfile from geonode.base.auth import get_or_create_token from geonode.tests.base import GeoNodeBaseTestSupport -from geonode.base.populate_test_data import all_public +from geonode.base.populate_test_data import ( + all_public, + create_models, + remove_models) class PermissionsApiTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + def setUp(self): - super(PermissionsApiTests, self).setUp() + super().setUp() self.user = 'admin' self.passwd = 'admin' - self.list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'layers'}) - all_public() self.perm_spec = {"users": {}, "groups": {}} def test_layer_get_list_unauth_all_public(self): @@ -61,8 +68,12 @@ def test_layer_get_list_unauth_all_public(self): Test that the correct number of layers are returned when the client is not logged in and all are public """ - - resp = self.api_client.get(self.list_url) + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) + resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)['objects']), 8) @@ -71,11 +82,16 @@ def test_layers_get_list_unauth_some_public(self): Test that if a layer is not public then not all are returned when the client is not logged in """ + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) layer = Layer.objects.all()[0] layer.set_permissions(self.perm_spec) - resp = self.api_client.get(self.list_url) + resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)['objects']), 7) @@ -84,79 +100,94 @@ def test_layers_get_list_auth_some_public(self): Test that if a layer is not public then all are returned if the client is not logged in """ + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) self.api_client.client.login(username=self.user, password=self.passwd) layer = Layer.objects.all()[0] layer.set_permissions(self.perm_spec) - resp = self.api_client.get(self.list_url) + resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 7) + self.assertEqual(len(self.deserialize(resp)['objects']), 8) def test_layer_get_list_layer_private_to_one_user(self): """ Test that if a layer is only visible by admin, then does not appear in the unauthenticated list nor in the list when logged is as bobby """ + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) perm_spec = {"users": {"admin": ['view_resourcebase']}, "groups": {}} layer = Layer.objects.all()[0] layer.set_permissions(perm_spec) - resp = self.api_client.get(self.list_url) + resp = self.api_client.get(list_url) self.assertEqual(len(self.deserialize(resp)['objects']), 7) self.api_client.client.login(username='bobby', password='bob') - resp = self.api_client.get(self.list_url) + resp = self.api_client.get(list_url) self.assertEqual(len(self.deserialize(resp)['objects']), 7) self.api_client.client.login(username=self.user, password=self.passwd) - resp = self.api_client.get(self.list_url) - self.assertEqual(len(self.deserialize(resp)['objects']), 7) + resp = self.api_client.get(list_url) + self.assertEqual(len(self.deserialize(resp)['objects']), 8) layer.is_published = False layer.save() # with resource publishing with self.settings(RESOURCE_PUBLISHING=True): - resp = self.api_client.get(self.list_url) + resp = self.api_client.get(list_url) self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) self.api_client.client.login(username='bobby', password='bob') - resp = self.api_client.get(self.list_url) + resp = self.api_client.get(list_url) self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) self.api_client.client.login(username=self.user, password=self.passwd) - resp = self.api_client.get(self.list_url) + resp = self.api_client.get(list_url) self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) def test_layer_get_detail_unauth_layer_not_public(self): """ Test that layer detail gives 404 when not public and not logged in """ + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) layer = Layer.objects.all()[0] layer.set_permissions(self.perm_spec) layer.clear_dirty_state() self.assertHttpNotFound(self.api_client.get( - f"{self.list_url + str(layer.id)}/")) + f"{list_url + str(layer.id)}/")) self.api_client.client.login(username=self.user, password=self.passwd) - resp = self.api_client.get(f"{self.list_url + str(layer.id)}/") + resp = self.api_client.get(f"{list_url + str(layer.id)}/") self.assertValidJSONResponse(resp) # with delayed security with self.settings(DELAYED_SECURITY_SIGNALS=True): if check_ogc_backend(geoserver.BACKEND_PACKAGE): - from geonode.security.utils import sync_geofence_with_guardian + from geonode.geoserver.security import sync_geofence_with_guardian sync_geofence_with_guardian(layer, self.perm_spec) self.assertTrue(layer.dirty_state) self.client.login(username=self.user, password=self.passwd) - resp = self.client.get(self.list_url) + resp = self.client.get(list_url) self.assertEqual(len(self.deserialize(resp)['objects']), 7) self.client.logout() - resp = self.client.get(self.list_url) + resp = self.client.get(list_url) self.assertEqual(len(self.deserialize(resp)['objects']), 7) from django.contrib.auth import get_user_model @@ -165,7 +196,7 @@ def test_layer_get_detail_unauth_layer_not_public(self): password='pbkdf2_sha256$12000$UE4gAxckVj4Z$N\ 6NbOXIQWWblfInIoq/Ta34FdRiPhawCIZ+sOO3YQs=') self.client.login(username='imnew', password='thepwd') - resp = self.client.get(self.list_url) + resp = self.client.get(list_url) self.assertEqual(len(self.deserialize(resp)['objects']), 7) def test_new_user_has_access_to_old_layers(self): @@ -176,8 +207,14 @@ def test_new_user_has_access_to_old_layers(self): password='pbkdf2_sha256$12000$UE4gAxckVj4Z$N\ 6NbOXIQWWblfInIoq/Ta34FdRiPhawCIZ+sOO3YQs=') + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) + self.api_client.client.login(username='imnew', password='thepwd') - resp = self.api_client.get(self.list_url) + resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) @@ -192,75 +229,209 @@ def test_new_user_has_access_to_old_layers(self): layer = Layer.objects.all()[0] layer.set_default_permissions() layer.refresh_from_db() - self.assertTrue(layer.dirty_state) + # self.assertTrue(layer.dirty_state) self.client.login(username=self.user, password=self.passwd) - resp = self.client.get(self.list_url) + resp = self.client.get(list_url) self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) self.client.logout() - resp = self.client.get(self.list_url) + resp = self.client.get(list_url) self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) self.client.login(username='imnew', password='thepwd') - resp = self.client.get(self.list_url) + resp = self.client.get(list_url) self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) finally: _ogc_geofence_enabled['default']['GEOFENCE_SECURITY_ENABLED'] = False - -class OAuthApiTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): - def setUp(self): - super(OAuthApiTests, self).setUp() - - self.user = 'admin' - self.passwd = 'admin' - self._user = get_user_model().objects.get(username=self.user) - self.token = get_or_create_token(self._user) - self.auth_header = f'Bearer {self.token}' - self.list_url = reverse( + @on_ogc_backend(geoserver.BACKEND_PACKAGE) + def test_outh_token(self): + user = 'admin' + _user = get_user_model().objects.get(username=user) + token = get_or_create_token(_user) + auth_header = f'Bearer {token}' + list_url = reverse( 'api_dispatch_list', kwargs={ 'api_name': 'api', 'resource_name': 'layers'}) - all_public() - self.perm_spec = {"users": {}, "groups": {}} - - @on_ogc_backend(geoserver.BACKEND_PACKAGE) - def test_outh_token(self): with self.settings(SESSION_EXPIRED_CONTROL_ENABLED=False, DELAYED_SECURITY_SIGNALS=False): # all public - resp = self.api_client.get(self.list_url) + resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) perm_spec = {"users": {"admin": ['view_resourcebase']}, "groups": {}} layer = Layer.objects.all()[0] layer.set_permissions(perm_spec) - resp = self.api_client.get(self.list_url) + resp = self.api_client.get(list_url) self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) - resp = self.api_client.get(self.list_url, authentication=self.auth_header) + resp = self.api_client.get(list_url, authentication=auth_header) self.assertGreaterEqual(len(self.deserialize(resp)['objects']), 7) layer.is_published = False layer.save() + @override_settings(API_LOCKDOWN=True) + def test_api_lockdown_false(self): + profiles_list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'profiles'}) -class SearchApiTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): + # test if results are returned for anonymous users if API_LOCKDOWN is set to False in settings + filter_url = profiles_list_url - """Test the search""" + with self.settings(API_LOCKDOWN=False): + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 9) - def setUp(self): - super(SearchApiTests, self).setUp() + @override_settings(API_LOCKDOWN=True) + def test_profiles_lockdown(self): + profiles_list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'profiles'}) + + filter_url = profiles_list_url + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 0) + + # now test with logged in user + self.api_client.client.login(username='bobby', password='bob') + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 9) + # Returns limitted info about other users + bobby = get_user_model().objects.get(username='bobby') + profiles = self.deserialize(resp)['objects'] + for profile in profiles: + if profile['username'] == 'bobby': + self.assertEquals(profile.get('email'), bobby.email) + else: + self.assertIsNone(profile.get('email')) - self.list_url = reverse( + @override_settings(API_LOCKDOWN=True) + def test_owners_lockdown(self): + owners_list_url = reverse( 'api_dispatch_list', kwargs={ 'api_name': 'api', - 'resource_name': 'layers'}) + 'resource_name': 'owners'}) + + filter_url = owners_list_url + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 0) + + # now test with logged in user + self.api_client.client.login(username='bobby', password='bob') + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 9) + # Returns limitted info about other users + bobby = get_user_model().objects.get(username='bobby') + owners = self.deserialize(resp)['objects'] + for owner in owners: + if owner['username'] == 'bobby': + self.assertEquals(owner.get('email'), bobby.email) + else: + self.assertIsNone(owner.get('email')) + self.assertIsNone(owner.get('first_name')) + + @override_settings(API_LOCKDOWN=True) + def test_groups_lockdown(self): + groups_list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'groups'}) + + filter_url = groups_list_url + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 0) + + # now test with logged in user + self.api_client.client.login(username='bobby', password='bob') + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 2) + + @override_settings(API_LOCKDOWN=True) + def test_regions_lockdown(self): + region_list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'regions'}) + + filter_url = region_list_url + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 0) + + self.api_client.client.login(username='bobby', password='bob') + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertTrue(len(self.deserialize(resp)['objects']) >= 200) + + @override_settings(API_LOCKDOWN=True) + def test_tags_lockdown(self): + tag_list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'keywords'}) + + filter_url = tag_list_url + + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 0) + + self.api_client.client.login(username='bobby', password='bob') + resp = self.api_client.get(filter_url) + self.assertValidJSONResponse(resp) + self.assertEqual(len(self.deserialize(resp)['objects']), 5) + + +class SearchApiTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): + + """Test the search""" + + # loading test thesausuri and initial data + fixtures = [ + 'initial_data.json', + 'group_test_data.json', + 'default_oauth_apps.json', + "test_thesaurus.json" + ] + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + + def setUp(self): + super().setUp() + self.norman = get_user_model().objects.get(username="norman") self.norman.groups.add(Group.objects.get(name='anonymous')) self.test_user = get_user_model().objects.get(username='test_user') @@ -337,16 +508,21 @@ def test_groups_filters(self): def test_category_filters(self): """Test category filtering""" + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) # check we get the correct layers number returnered filtering on one # and then two different categories - filter_url = f"{self.list_url}?category__identifier=location" + filter_url = f"{list_url}?category__identifier=location" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)['objects']), 3) - filter_url = f"{self.list_url}?category__identifier__in=location&category__identifier__in=biota" + filter_url = f"{list_url}?category__identifier__in=location&category__identifier__in=biota" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) @@ -354,16 +530,21 @@ def test_category_filters(self): def test_tag_filters(self): """Test keywords filtering""" + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) # check we get the correct layers number returnered filtering on one # and then two different keywords - filter_url = f"{self.list_url}?keywords__slug=layertagunique" + filter_url = f"{list_url}?keywords__slug=layertagunique" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)['objects']), 1) - filter_url = f"{self.list_url}?keywords__slug__in=layertagunique&keywords__slug__in=populartag" + filter_url = f"{list_url}?keywords__slug__in=layertagunique&keywords__slug__in=populartag" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) @@ -371,16 +552,21 @@ def test_tag_filters(self): def test_owner_filters(self): """Test owner filtering""" + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) # check we get the correct layers number returnered filtering on one # and then two different owners - filter_url = f"{self.list_url}?owner__username=user1" + filter_url = f"{list_url}?owner__username=user1" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)['objects']), 1) - filter_url = f"{self.list_url}?owner__username__in=user1&owner__username__in=foo" + filter_url = f"{list_url}?owner__username__in=user1&owner__username__in=foo" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) @@ -388,10 +574,15 @@ def test_owner_filters(self): def test_title_filter(self): """Test title filtering""" + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) # check we get the correct layers number returnered filtering on the # title - filter_url = f"{self.list_url}?title=layer2" + filter_url = f"{list_url}?title=layer2" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) @@ -399,6 +590,11 @@ def test_title_filter(self): def test_date_filter(self): """Test date filtering""" + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) # check we get the correct layers number returnered filtering on the # dates @@ -410,21 +606,21 @@ def to_date(val): return val.date().strftime(fstring) d1 = to_date(now - step) - filter_url = f"{self.list_url}?date__exact={d1}" + filter_url = f"{list_url}?date__exact={d1}" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)['objects']), 0) d3 = to_date(now - (3 * step)) - filter_url = f"{self.list_url}?date__gte={d3}" + filter_url = f"{list_url}?date__gte={d3}" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)['objects']), 3) d4 = to_date(now - (4 * step)) - filter_url = f"{self.list_url}?date__range={d4},{to_date(now)}" + filter_url = f"{list_url}?date__range={d4},{to_date(now)}" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) @@ -432,180 +628,159 @@ def to_date(val): def test_extended_text_filter(self): """Test that the extended text filter works as expected""" - filter_url = f"{self.list_url}?title__icontains=layer2&abstract__icontains=layer2&purpose__icontains=layer2&f_method=or" + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'layers'}) + + filter_url = f"{list_url}?title__icontains=layer2&abstract__icontains=layer2&purpose__icontains=layer2&f_method=or" resp = self.api_client.get(filter_url) self.assertValidJSONResponse(resp) self.assertEqual(len(self.deserialize(resp)['objects']), 1) - -# noinspection DuplicatedCode -@override_settings(API_LOCKDOWN=True) -class LockdownApiTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): - - """Test the api lockdown functionality""" - - def setUp(self): - super(LockdownApiTests, self).setUp() - self.profiles_list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'profiles'}) - self.groups_list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'groups'}) - self.owners_list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'owners'}) - self.tag_list_url = reverse( + def test_the_api_should_return_all_layers_with_metadata_false(self): + list_url = reverse( 'api_dispatch_list', kwargs={ 'api_name': 'api', - 'resource_name': 'keywords'}) - self.region_list_url = reverse( + 'resource_name': 'layers'}) + user = get_user_model().objects.get(username="admin") + token = get_or_create_token(user) + auth_header = f'Bearer {token}' + + resp = self.api_client.get(list_url, authentication=auth_header) + self.assertValidJSONResponse(resp) + self.assertEqual(8, resp.json()["meta"]["total_count"]) + + def test_the_api_should_return_all_layers_with_metadata_true(self): + list_url = reverse( 'api_dispatch_list', kwargs={ 'api_name': 'api', - 'resource_name': 'regions'}) - self.bobby = get_user_model().objects.get(username='bobby') - - def test_api_lockdown_false(self): - # test if results are returned for anonymous users if API_LOCKDOWN is set to False in settings - filter_url = self.profiles_list_url - - with self.settings(API_LOCKDOWN=False): - resp = self.api_client.get(filter_url) - self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 9) - - def test_profiles_lockdown(self): - filter_url = self.profiles_list_url - resp = self.api_client.get(filter_url) - self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + 'resource_name': 'layers'}) + user = get_user_model().objects.get(username="admin") + token = get_or_create_token(user) + auth_header = f'Bearer {token}' - # now test with logged in user - self.api_client.client.login(username='bobby', password='bob') - resp = self.api_client.get(filter_url) + url = f"{list_url}?metadata_only=True" + resp = self.api_client.get(url, authentication=auth_header) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 9) - # Returns limitted info about other users - profiles = self.deserialize(resp)['objects'] - for profile in profiles: - if profile['username'] == 'bobby': - self.assertEquals(profile.get('email'), self.bobby.email) - else: - self.assertIsNone(profile.get('email')) - - def test_owners_lockdown(self): - filter_url = self.owners_list_url + self.assertEqual(1, resp.json()["meta"]["total_count"]) - resp = self.api_client.get(filter_url) - self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + def test_the_api_should_return_all_documents_with_metadata_false(self): + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'documents'}) - # now test with logged in user - self.api_client.client.login(username='bobby', password='bob') - resp = self.api_client.get(filter_url) + resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 9) - # Returns limitted info about other users - owners = self.deserialize(resp)['objects'] - for owner in owners: - if owner['username'] == 'bobby': - self.assertEquals(owner.get('email'), self.bobby.email) - else: - self.assertIsNone(owner.get('email')) - self.assertIsNone(owner.get('first_name')) - - def test_groups_lockdown(self): - filter_url = self.groups_list_url + self.assertEqual(resp.json()["meta"]["total_count"], 9) - resp = self.api_client.get(filter_url) - self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + def test_the_api_should_return_all_documents_with_metadata_true(self): + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'documents'}) - # now test with logged in user - self.api_client.client.login(username='bobby', password='bob') - resp = self.api_client.get(filter_url) + url = f"{list_url}?metadata_only=True" + resp = self.api_client.get(url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 2) - - def test_regions_lockdown(self): - filter_url = self.region_list_url + self.assertEqual(resp.json()["meta"]["total_count"], 1) - resp = self.api_client.get(filter_url) - self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + def test_the_api_should_return_all_maps_with_metadata_false(self): + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'maps'}) - self.api_client.client.login(username='bobby', password='bob') - resp = self.api_client.get(filter_url) + resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) - self.assertTrue(len(self.deserialize(resp)['objects']) >= 200) - - def test_tags_lockdown(self): - filter_url = self.tag_list_url + self.assertEqual(resp.json()["meta"]["total_count"], 9) - resp = self.api_client.get(filter_url) - self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 0) + def test_the_api_should_return_all_maps_with_metadata_true(self): + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'maps'}) - self.api_client.client.login(username='bobby', password='bob') - resp = self.api_client.get(filter_url) + url = f"{list_url}?metadata_only=True" + resp = self.api_client.get(url) self.assertValidJSONResponse(resp) - self.assertEqual(len(self.deserialize(resp)['objects']), 5) - - -class ThesaurusKeywordResourceTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): - - # loading test thesausuri - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json', - "test_thesaurus.json" - ] - - def setUp(self): - super(ThesaurusKeywordResourceTests, self).setUp() - all_public() - self.list_url = reverse("api_dispatch_list", kwargs={"api_name": "api", "resource_name": "thesaurus/keywords"}) + self.assertEqual(resp.json()["meta"]["total_count"], 1) def test_api_will_return_a_valid_json_response(self): - resp = self.api_client.get(self.list_url) + list_url = reverse( + "api_dispatch_list", + kwargs={ + "api_name": "api", + "resource_name": "thesaurus/keywords"}) + + resp = self.api_client.get(list_url) self.assertValidJSONResponse(resp) def test_will_return_empty_if_the_thesaurus_does_not_exists(self): - url = f"{self.list_url}?thesaurus=invalid-identifier" + list_url = reverse( + "api_dispatch_list", + kwargs={ + "api_name": "api", + "resource_name": "thesaurus/keywords"}) + + url = f"{list_url}?thesaurus=invalid-identifier" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) self.assertEqual(resp.json()["meta"]["total_count"], 0) def test_will_return_keywords_for_the_selected_thesaurus_if_exists(self): - url = f"{self.list_url}?thesaurus=inspire-theme" + list_url = reverse( + "api_dispatch_list", + kwargs={ + "api_name": "api", + "resource_name": "thesaurus/keywords"}) + + url = f"{list_url}?thesaurus=inspire-theme" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) self.assertEqual(resp.json()["meta"]["total_count"], 36) def test_will_return_empty_if_the_alt_label_does_not_exists(self): - url = f"{self.list_url}?alt_label=invalid-alt_label" + list_url = reverse( + "api_dispatch_list", + kwargs={ + "api_name": "api", + "resource_name": "thesaurus/keywords"}) + + url = f"{list_url}?alt_label=invalid-alt_label" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) self.assertEqual(resp.json()["meta"]["total_count"], 0) def test_will_return_keywords_for_the_selected_alt_label_if_exists(self): - url = f"{self.list_url}?alt_label=ac" + list_url = reverse( + "api_dispatch_list", + kwargs={ + "api_name": "api", + "resource_name": "thesaurus/keywords"}) + + url = f"{list_url}?alt_label=ac" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) self.assertEqual(resp.json()["meta"]["total_count"], 1) def test_will_return_empty_if_the_kaywordId_does_not_exists(self): - url = f"{self.list_url}?id=12365478954862" + list_url = reverse( + "api_dispatch_list", + kwargs={ + "api_name": "api", + "resource_name": "thesaurus/keywords"}) + + url = f"{list_url}?id=12365478954862" resp = self.api_client.get(url) print(self.deserialize(resp)) self.assertValidJSONResponse(resp) @@ -613,8 +788,14 @@ def test_will_return_empty_if_the_kaywordId_does_not_exists(self): @patch("geonode.api.api.get_language") def test_will_return_expected_keyword_label_for_existing_lang(self, lang): + list_url = reverse( + "api_dispatch_list", + kwargs={ + "api_name": "api", + "resource_name": "thesaurus/keywords"}) + lang.return_value = "de" - url = f"{self.list_url}?thesaurus=inspire-theme" + url = f"{list_url}?thesaurus=inspire-theme" resp = self.api_client.get(url) # the german translations exists, for the other labels, the alt_label will be used expected_labels = [ @@ -629,8 +810,14 @@ def test_will_return_expected_keyword_label_for_existing_lang(self, lang): @patch("geonode.api.api.get_language") def test_will_return_default_keyword_label_for_not_existing_lang(self, lang): + list_url = reverse( + "api_dispatch_list", + kwargs={ + "api_name": "api", + "resource_name": "thesaurus/keywords"}) + lang.return_value = "ke" - url = f"{self.list_url}?thesaurus=inspire-theme" + url = f"{list_url}?thesaurus=inspire-theme" resp = self.api_client.get(url) # no translations exists, the alt_label will be used for all keywords expected_labels = [ @@ -643,146 +830,62 @@ def test_will_return_default_keyword_label_for_not_existing_lang(self, lang): self.assertValidJSONResponse(resp) self.assertListEqual(expected_labels, actual_labels) - -class LayerResourceTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] - - def setUp(self): - super(LayerResourceTests, self).setUp() - self.user = get_user_model().objects.get(username="admin") - self.list_url = reverse( + def test_the_api_should_return_all_map_categories_with_metadata_false(self): + list_url = reverse( 'api_dispatch_list', kwargs={ 'api_name': 'api', - 'resource_name': 'layers'}) - all_public() - self.token = get_or_create_token(self.user) - self.auth_header = f'Bearer {self.token}' - - def test_the_api_should_return_all_layers_with_metadata_false(self): - - resp = self.api_client.get(self.list_url, authentication=self.auth_header) - self.assertValidJSONResponse(resp) - self.assertEqual(8, resp.json()["meta"]["total_count"]) - - def test_the_api_should_return_all_layers_with_metadata_true(self): - - url = f"{self.list_url}?metadata_only=True" - resp = self.api_client.get(url, authentication=self.auth_header) - self.assertValidJSONResponse(resp) - self.assertEqual(1, resp.json()["meta"]["total_count"]) - - -class DocumentResourceTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] - - def setUp(self): - super(DocumentResourceTests, self).setUp() - all_public() - self.user = get_user_model().objects.get(username="admin") - self.list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'documents'}) - - def test_the_api_should_return_all_documents_with_metadata_false(self): - resp = self.api_client.get(self.list_url) - self.assertValidJSONResponse(resp) - self.assertEqual(resp.json()["meta"]["total_count"], 9) - - def test_the_api_should_return_all_documents_with_metadata_true(self): - url = f"{self.list_url}?metadata_only=True" - resp = self.api_client.get(url) - self.assertValidJSONResponse(resp) - self.assertEqual(resp.json()["meta"]["total_count"], 1) - - -class MapResourceTests(ResourceTestCaseMixin, GeoNodeBaseTestSupport): - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] - - def setUp(self): - super(MapResourceTests, self).setUp() - all_public() - self.user = get_user_model().objects.get(username="admin") - self.list_url = reverse( - 'api_dispatch_list', - kwargs={ - 'api_name': 'api', - 'resource_name': 'maps'}) - - def test_the_api_should_return_all_maps_with_metadata_false(self): - resp = self.api_client.get(self.list_url) - self.assertValidJSONResponse(resp) - self.assertEqual(resp.json()["meta"]["total_count"], 9) + 'resource_name': 'categories'}) - def test_the_api_should_return_all_maps_with_metadata_true(self): - url = f"{self.list_url}?metadata_only=True" + url = f"{list_url}?type=map" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) - self.assertEqual(resp.json()["meta"]["total_count"], 1) - - -class TopicCategoryResourceTest(ResourceTestCaseMixin, GeoNodeBaseTestSupport): - fixtures = [ - 'initial_data.json', - 'group_test_data.json', - 'default_oauth_apps.json' - ] + actual = sum([x['count'] for x in resp.json()['objects']]) + self.assertEqual(9, actual) - def setUp(self): - super(TopicCategoryResourceTest, self).setUp() - self.user = get_user_model().objects.get(username="admin") - self.list_url = reverse( + def test_the_api_should_return_all_map_categories_with_metadata_true(self): + list_url = reverse( 'api_dispatch_list', kwargs={ 'api_name': 'api', 'resource_name': 'categories'}) - def test_the_api_should_return_all_maps_with_metadata_false(self): - url = f"{self.list_url}?type=map" - resp = self.api_client.get(url) - self.assertValidJSONResponse(resp) - actual = sum([x['count'] for x in resp.json()['objects']]) - self.assertEqual(9, actual) - - def test_the_api_should_return_all_maps_with_metadata_true(self): x = Map.objects.get(title='map metadata true') x.metadata_only = False x.save() - url = f"{self.list_url}?type=map" + url = f"{list_url}?type=map" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) # by adding a new layer, the total should increase actual = sum([x['count'] for x in resp.json()['objects']]) self.assertEqual(10, actual) - def test_the_api_should_return_all_document_with_metadata_false(self): - url = f"{self.list_url}?type=document" + def test_the_api_should_return_all_document_categories_with_metadata_false(self): + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'categories'}) + + url = f"{list_url}?type=document" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) actual = sum([x['count'] for x in resp.json()['objects']]) - self.assertEqual(9, actual) + self.assertEqual(0, actual) + + def test_the_api_should_return_all_document_categories_with_metadata_true(self): + list_url = reverse( + 'api_dispatch_list', + kwargs={ + 'api_name': 'api', + 'resource_name': 'categories'}) - def test_the_api_should_return_all_document_with_metadata_true(self): x = Document.objects.get(title='doc metadata true') x.metadata_only = False x.save() - url = f"{self.list_url}?type=document" + url = f"{list_url}?type=document" resp = self.api_client.get(url) self.assertValidJSONResponse(resp) # by adding a new layer, the total should increase actual = sum([x['count'] for x in resp.json()['objects']]) - self.assertEqual(10, actual) + self.assertEqual(0, actual) diff --git a/geonode/api/urls.py b/geonode/api/urls.py index 9eac64c7e9f..c955b00c492 100644 --- a/geonode/api/urls.py +++ b/geonode/api/urls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/api/views.py b/geonode/api/views.py index 824c5f49d21..bd340597e3d 100644 --- a/geonode/api/views.py +++ b/geonode/api/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/apps.py b/geonode/apps.py index a03d81df3cf..efa25ca5b53 100644 --- a/geonode/apps.py +++ b/geonode/apps.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo @@ -33,5 +32,5 @@ class AppConfig(BaseAppConfig): label = "geonode" def ready(self): - super(AppConfig, self).ready() + super().ready() run_setup_hooks() diff --git a/geonode/base/__init__.py b/geonode/base/__init__.py index 6938ea6f316..01e8a2f5a2e 100644 --- a/geonode/base/__init__.py +++ b/geonode/base/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/base/admin.py b/geonode/base/admin.py index a81bbcd01dc..7ae27e3a762 100755 --- a/geonode/base/admin.py +++ b/geonode/base/admin.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -281,7 +280,7 @@ class ThesaurusAdmin(admin.ModelAdmin): ordering = ('identifier',) def get_urls(self): - urls = super(ThesaurusAdmin, self).get_urls() + urls = super().get_urls() my_urls = [ path('importrdf/', self.import_rdf, name="base_thesaurus_importrdf") ] @@ -378,5 +377,13 @@ class ResourceBaseAdminForm(autocomplete.FutureModelForm): keywords = TagField(widget=TaggitSelect2Custom('autocomplete_hierachical_keyword')) + def delete_queryset(self, request, queryset): + """ + We need to invoke the 'ResourceBase.delete' method even when deleting + through the admin batch action + """ + for obj in queryset: + obj.delete() + class Meta: pass diff --git a/geonode/base/api/__init__.py b/geonode/base/api/__init__.py index fe4e643c905..0044807c781 100644 --- a/geonode/base/api/__init__.py +++ b/geonode/base/api/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/base/api/filters.py b/geonode/base/api/filters.py index c540585bc29..51850744bfd 100644 --- a/geonode/base/api/filters.py +++ b/geonode/base/api/filters.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo @@ -52,7 +51,7 @@ class FavoriteFilter(BaseFilterBackend): def filter_queryset(self, request, queryset, _): if strtobool(request.query_params.get("favorite", 'False')): - ctype = list(set([r.resource_type for r in queryset])) + ctype = list({r.resource_type for r in queryset}) return queryset.filter( pk__in=Subquery( Favorite.objects.values_list("object_id", flat=True) diff --git a/geonode/base/api/pagination.py b/geonode/base/api/pagination.py index 2f73c1aad70..0b5675e5c83 100644 --- a/geonode/base/api/pagination.py +++ b/geonode/base/api/pagination.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/base/api/permissions.py b/geonode/base/api/permissions.py index a97c84916b8..5efc48bcc0b 100644 --- a/geonode/base/api/permissions.py +++ b/geonode/base/api/permissions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index 229be726f37..5d674e81db2 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo @@ -17,6 +16,7 @@ # along with this program. If not, see . # ######################################################################### +import json from urllib.parse import urljoin from django.contrib.auth.models import Group @@ -59,7 +59,7 @@ class BaseDynamicModelSerializer(DynamicModelSerializer): def to_representation(self, instance): - data = super(BaseDynamicModelSerializer, self).to_representation(instance) + data = super().to_representation(instance) try: path = reverse(self.Meta.view_name) if not path.endswith('/'): @@ -169,7 +169,7 @@ class AvatarUrlField(DynamicComputedField): def __init__(self, avatar_size, **kwargs): self.avatar_size = avatar_size - super(AvatarUrlField, self).__init__(**kwargs) + super().__init__(**kwargs) def get_attribute(self, instance): return build_absolute_uri(avatar_url(instance, self.avatar_size)) @@ -178,7 +178,7 @@ def get_attribute(self, instance): class EmbedUrlField(DynamicComputedField): def __init__(self, **kwargs): - super(EmbedUrlField, self).__init__(**kwargs) + super().__init__(**kwargs) def get_attribute(self, instance): _instance = instance.get_real_instance() @@ -191,7 +191,7 @@ def get_attribute(self, instance): class DetailUrlField(DynamicComputedField): def __init__(self, **kwargs): - super(DetailUrlField, self).__init__(**kwargs) + super().__init__(**kwargs) def get_attribute(self, instance): return build_absolute_uri(instance.detail_url) @@ -200,7 +200,7 @@ def get_attribute(self, instance): class ThumbnailUrlField(DynamicComputedField): def __init__(self, **kwargs): - super(ThumbnailUrlField, self).__init__(**kwargs) + super().__init__(**kwargs) def get_attribute(self, instance): thumbnail_url = instance.thumbnail_url @@ -236,7 +236,7 @@ class ContactRoleField(DynamicComputedField): def __init__(self, contat_type, **kwargs): self.contat_type = contat_type - super(ContactRoleField, self).__init__(**kwargs) + super().__init__(**kwargs) def get_attribute(self, instance): return getattr(instance, self.contat_type) @@ -245,11 +245,34 @@ def to_representation(self, value): return UserSerializer(embed=True, many=False).to_representation(value) +class DataBlobField(DynamicRelationField): + + def value_to_string(self, obj): + value = self.value_from_object(obj) + return self.get_prep_value(value) + + +class DataBlobSerializer(DynamicModelSerializer): + + class Meta: + model = ResourceBase + fields = ('pk', 'blob') + + def to_internal_value(self, data): + return data + + def to_representation(self, value): + data = ResourceBase.objects.filter(id=value) + if data.exists() and data.count() == 1: + return data.get().blob + return {} + + class ResourceBaseSerializer(BaseDynamicModelSerializer): def __init__(self, *args, **kwargs): # Instantiate the superclass normally - super(ResourceBaseSerializer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.fields['pk'] = serializers.CharField(read_only=True) self.fields['uuid'] = serializers.CharField(read_only=True) @@ -294,6 +317,7 @@ def __init__(self, *args, **kwargs): self.fields['raw_data_quality_statement'] = serializers.CharField(read_only=True) self.fields['metadata_only'] = serializers.BooleanField() self.fields['processed'] = serializers.BooleanField(read_only=True) + self.fields['state'] = serializers.CharField(read_only=True) self.fields['embed_url'] = EmbedUrlField() self.fields['thumbnail_url'] = ThumbnailUrlField() @@ -309,6 +333,7 @@ def __init__(self, *args, **kwargs): LicenseSerializer, embed=True, many=False) self.fields['spatial_representation_type'] = DynamicRelationField( SpatialRepresentationTypeSerializer, embed=True, many=False) + self.fields['storeType'] = serializers.CharField(read_only=True) class Meta: model = ResourceBase @@ -326,16 +351,28 @@ class Meta: 'popular_count', 'share_count', 'rating', 'featured', 'is_published', 'is_approved', 'detail_url', 'embed_url', 'created', 'last_updated', 'raw_abstract', 'raw_purpose', 'raw_constraints_other', - 'raw_supplemental_information', 'raw_data_quality_statement', 'metadata_only', 'processed' + 'raw_supplemental_information', 'raw_data_quality_statement', 'metadata_only', 'processed', 'state', + 'data', 'storetype' # TODO # csw_typename, csw_schema, csw_mdsource, csw_insert_date, csw_type, csw_anytext, csw_wkt_geometry, # metadata_uploaded, metadata_uploaded_preserve, metadata_xml, # users_geolimits, groups_geolimits ) + def to_internal_value(self, data): + if isinstance(data, str): + data = json.loads(data) + if 'data' in data: + _data = data.pop('data') + if self.is_valid(): + data['blob'] = _data + if 'storeType' in data: + data['storetype'] = data.pop('storeType') + return data + def to_representation(self, instance): request = self.context.get('request') - data = super(ResourceBaseSerializer, self).to_representation(instance) + data = super().to_representation(instance) if request: data['perms'] = instance.get_user_perms(request.user).union( instance.get_self_resource().get_user_perms(request.user) @@ -343,6 +380,8 @@ def to_representation(self, instance): if not request.user.is_anonymous: favorite = Favorite.objects.filter(user=request.user, object_id=instance.pk).count() data['favorite'] = favorite > 0 + if 'storetype' in data: + data['storeType'] = data.pop('storetype') # Adding links to resource_base api obj_id = data.get('pk', None) if obj_id: @@ -363,6 +402,16 @@ def to_representation(self, instance): data['links'] = dehydrated return data + """ + - Deferred / not Embedded --> ?include[]=data + """ + data = DataBlobField( + DataBlobSerializer, + source='id', + many=False, + embed=False, + deferred=True) + class FavoriteSerializer(DynamicModelSerializer): resource = serializers.SerializerMethodField() @@ -373,7 +422,7 @@ class Meta: fields = 'resource', def to_representation(self, value): - data = super(FavoriteSerializer, self).to_representation(value) + data = super().to_representation(value) return data['resource'] def get_resource(self, instance): @@ -391,7 +440,7 @@ def to_representation(self, instance): 'type_filter': request.query_params.get('type'), 'title_filter': request.query_params.get('title__icontains') } - data = super(BaseResourceCountSerializer, self).to_representation(instance) + data = super().to_representation(instance) count_filter = {self.Meta.count_type: instance} data['count'] = get_resources_with_perms( request.user, filter_options).filter(**count_filter).count() diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index 18fd665a847..03e616b3d04 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -18,24 +17,21 @@ # ######################################################################### import logging + from uuid import uuid4 from PIL import Image from io import BytesIO from unittest.mock import patch from urllib.parse import urljoin -import django from django.urls import reverse from django.core.files import File -from django.conf.urls import url, include -from django.views.generic import TemplateView from django.contrib.auth import get_user_model -from django.views.i18n import JavaScriptCatalog -from rest_framework.test import APITestCase, URLPatternsTestCase +from rest_framework.test import APITestCase from guardian.shortcuts import get_anonymous_user -from geonode.api.urls import router +from geonode.base import enumerations from geonode.base.models import ( CuratedThumbnail, HierarchicalKeyword, @@ -45,14 +41,8 @@ ThesaurusKeyword, ) -from geonode import geoserver from geonode.favorite.models import Favorite -from geonode.utils import check_ogc_backend -from geonode.services.views import services -from geonode.maps.views import map_embed from geonode.layers.models import Layer -from geonode.layers.views import layer_embed, layer_detail -from geonode.geoapps.views import geoapp_edit from geonode.base.utils import build_absolute_uri from geonode.base.populate_test_data import create_models from geonode.security.utils import get_resources_with_perms @@ -62,7 +52,7 @@ test_image = Image.new('RGBA', size=(50, 50), color=(155, 0, 0)) -class BaseApiTests(APITestCase, URLPatternsTestCase): +class BaseApiTests(APITestCase): fixtures = [ 'initial_data.json', @@ -71,56 +61,6 @@ class BaseApiTests(APITestCase, URLPatternsTestCase): "test_thesaurus.json" ] - urlpatterns = [ - url(r'^home/$', - TemplateView.as_view(template_name='index.html'), - name='home'), - url(r'^help/$', - TemplateView.as_view(template_name='help.html'), - name='help'), - url(r"^account/", include("allauth.urls")), - url(r'^people/', include('geonode.people.urls')), - url(r'^api/v2/', include(router.urls)), - url(r'^api/v2/', include('geonode.api.urls')), - url(r'^api/v2/api-auth/', include('rest_framework.urls', namespace='geonode_rest_framework')), - url(r'^$', - TemplateView.as_view(template_name='layers/layer_list.html'), - {'facet_type': 'layers', 'is_layer': True}, - name='layer_browse'), - url(r'^$', - TemplateView.as_view(template_name='maps/map_list.html'), - {'facet_type': 'maps', 'is_map': True}, - name='maps_browse'), - url(r'^$', - TemplateView.as_view(template_name='documents/document_list.html'), - {'facet_type': 'documents', 'is_document': True}, - name='document_browse'), - url(r'^$', - TemplateView.as_view(template_name='groups/group_list.html'), - name='group_list'), - url(r'^search/$', - TemplateView.as_view(template_name='search/search.html'), - name='search'), - url(r'^$', services, name='services'), - url(r'^invitations/', include( - 'geonode.invitations.urls', namespace='geonode.invitations')), - url(r'^i18n/', include(django.conf.urls.i18n), name="i18n"), - url(r'^jsi18n/$', JavaScriptCatalog.as_view(), {}, name='javascript-catalog'), - url(r'^(?P[^/]+)/embed$', map_embed, name='map_embed'), - url(r'^(?P[^/]+)/embed$', layer_embed, name='layer_embed'), - url(r'^(?P[^/]+)/embed$', geoapp_edit, {'template': 'apps/app_embed.html'}, name='geoapp_embed'), - url(r'^(?P[^/]*)$', layer_detail, name="layer_detail"), - ] - - if check_ogc_backend(geoserver.BACKEND_PACKAGE): - from geonode.geoserver.views import layer_acls, resolve_user - urlpatterns += [ - url(r'^acls/?$', layer_acls, name='layer_acls'), - url(r'^acls_dep/?$', layer_acls, name='layer_acls_dep'), - url(r'^resolve_user/?$', resolve_user, name='layer_resolve_user'), - url(r'^resolve_user_dep/?$', resolve_user, name='layer_resolve_user_dep'), - ] - def setUp(self): create_models(b'document') create_models(b'map') @@ -254,7 +194,7 @@ def test_base_resources(self): response = self.client.get(url, format='json') self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 18) + self.assertEqual(response.data['total'], 26) # Pagination self.assertEqual(len(response.data['resources']), 10) @@ -263,7 +203,7 @@ def test_base_resources(self): response = self.client.get(url, format='json') self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 18) + self.assertEqual(response.data['total'], 26) # response has link to the response self.assertTrue('link' in response.data['resources'][0].keys()) # Pagination @@ -275,7 +215,7 @@ def test_base_resources(self): response = self.client.get(url, format='json') self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 18) + self.assertEqual(response.data['total'], 26) # Pagination self.assertEqual(len(response.data['resources']), 10) logger.debug(response.data) @@ -285,7 +225,7 @@ def test_base_resources(self): response = self.client.get(url, format='json') self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 18) + self.assertEqual(response.data['total'], 26) # Pagination self.assertEqual(len(response.data['resources']), 10) logger.debug(response.data) @@ -297,7 +237,7 @@ def test_base_resources(self): response = self.client.get(f"{url}?page_size=17", format='json') self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 18) + self.assertEqual(response.data['total'], 26) # Pagination self.assertEqual(len(response.data['resources']), 17) @@ -305,6 +245,7 @@ def test_base_resources(self): resource = ResourceBase.objects.filter(owner__username='bobby').first() # Admin response = self.client.get(f"{url}/{resource.id}/", format='json') + self.assertEqual(response.data['resource']['state'], enumerations.STATE_PROCESSED) self.assertTrue('change_resourcebase' in list(response.data['resource']['perms'])) # Annonymous self.assertIsNone(self.client.logout()) @@ -328,7 +269,7 @@ def test_delete_user_with_resource(self): alternate='Test Remove User', uuid=str(uuid4()), owner=owner, - storeType='coverageStore', + storetype='raster', category=TopicCategory.objects.get(identifier='elevation') ).save() # Delete user and check if default user is updated @@ -410,9 +351,9 @@ def test_filter_resources(self): f"{url}?filter{{category.identifier}}=elevation", format='json') self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) - self.assertEqual(response.data['total'], 9) + self.assertEqual(response.data['total'], 6) # Pagination - self.assertEqual(len(response.data['resources']), 9) + self.assertEqual(len(response.data['resources']), 6) # Extent Filter response = self.client.get(f"{url}?page_size=26&extent=-180,-90,180,90", format='json') diff --git a/geonode/base/api/urls.py b/geonode/base/api/urls.py index 69465c7602c..19a7336b1b7 100644 --- a/geonode/base/api/urls.py +++ b/geonode/base/api/urls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index b839ef7e456..7dcff30f89b 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo @@ -19,20 +18,22 @@ ######################################################################### from django.apps import apps from django.conf import settings -from django.contrib.auth import get_user_model from django.db.models import Subquery +from django.contrib.auth import get_user_model from drf_spectacular.utils import extend_schema from dynamic_rest.viewsets import DynamicModelViewSet, WithDynamicViewSetMixin from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter -from rest_framework.mixins import ListModelMixin, RetrieveModelMixin -from rest_framework.viewsets import GenericViewSet +from oauth2_provider.contrib.rest_framework import OAuth2Authentication + from rest_framework.response import Response from rest_framework.decorators import action +from rest_framework.viewsets import GenericViewSet +from rest_framework.mixins import ListModelMixin, RetrieveModelMixin from rest_framework.permissions import AllowAny, IsAuthenticated, IsAuthenticatedOrReadOnly from rest_framework.authentication import SessionAuthentication, BasicAuthentication -from oauth2_provider.contrib.rest_framework import OAuth2Authentication + from geonode.favorite.models import Favorite from geonode.base.models import HierarchicalKeyword, Region, ResourceBase, TopicCategory, ThesaurusKeyword from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter, FavoriteFilter @@ -230,7 +231,7 @@ class ResourceBaseViewSet(DynamicModelViewSet): DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, ExtentFilter, ResourceBasePermissionsFilter, FavoriteFilter ] - queryset = ResourceBase.objects.all() + queryset = ResourceBase.objects.all().order_by('-date') serializer_class = ResourceBaseSerializer pagination_class = GeoNodeApiPagination diff --git a/geonode/base/auth.py b/geonode/base/auth.py index 7d83ad087ab..56861c9550d 100644 --- a/geonode/base/auth.py +++ b/geonode/base/auth.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2019 OSGeo diff --git a/geonode/base/enumerations.py b/geonode/base/enumerations.py index e0001052df2..856c7a90e42 100644 --- a/geonode/base/enumerations.py +++ b/geonode/base/enumerations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -437,36 +436,60 @@ ('zul', 'Zulu'), ) -CHARSETS = (('', 'None/Unknown'), - ('UTF-8', 'UTF-8/Unicode'), - ('ISO-8859-1', 'Latin1/ISO-8859-1'), - ('ISO-8859-2', 'Latin2/ISO-8859-2'), - ('ISO-8859-3', 'Latin3/ISO-8859-3'), - ('ISO-8859-4', 'Latin4/ISO-8859-4'), - ('ISO-8859-5', 'Latin5/ISO-8859-5'), - ('ISO-8859-6', 'Latin6/ISO-8859-6'), - ('ISO-8859-7', 'Latin7/ISO-8859-7'), - ('ISO-8859-8', 'Latin8/ISO-8859-8'), - ('ISO-8859-9', 'Latin9/ISO-8859-9'), - ('ISO-8859-10', 'Latin10/ISO-8859-10'), - ('ISO-8859-13', 'Latin13/ISO-8859-13'), - ('ISO-8859-14', 'Latin14/ISO-8859-14'), - ('ISO8859-15', 'Latin15/ISO-8859-15'), - ('Big5', 'BIG5'), - ('EUC-JP', 'EUC-JP'), - ('EUC-KR', 'EUC-KR'), - ('GBK', 'GBK'), - ('GB18030', 'GB18030'), - ('Shift_JIS', 'Shift_JIS'), - ('KOI8-R', 'KOI8-R'), - ('KOI8-U', 'KOI8-U'), - ('cp874', 'Windows CP874'), - ('windows-1250', 'Windows CP1250'), - ('windows-1251', 'Windows CP1251'), - ('windows-1252', 'Windows CP1252'), - ('windows-1253', 'Windows CP1253'), - ('windows-1254', 'Windows CP1254'), - ('windows-1255', 'Windows CP1255'), - ('windows-1256', 'Windows CP1256'), - ('windows-1257', 'Windows CP1257'), - ('windows-1258', 'Windows CP1258')) +CHARSETS = ( + ('', 'None/Unknown'), + ('UTF-8', 'UTF-8/Unicode'), + ('ISO-8859-1', 'Latin1/ISO-8859-1'), + ('ISO-8859-2', 'Latin2/ISO-8859-2'), + ('ISO-8859-3', 'Latin3/ISO-8859-3'), + ('ISO-8859-4', 'Latin4/ISO-8859-4'), + ('ISO-8859-5', 'Latin5/ISO-8859-5'), + ('ISO-8859-6', 'Latin6/ISO-8859-6'), + ('ISO-8859-7', 'Latin7/ISO-8859-7'), + ('ISO-8859-8', 'Latin8/ISO-8859-8'), + ('ISO-8859-9', 'Latin9/ISO-8859-9'), + ('ISO-8859-10', 'Latin10/ISO-8859-10'), + ('ISO-8859-13', 'Latin13/ISO-8859-13'), + ('ISO-8859-14', 'Latin14/ISO-8859-14'), + ('ISO8859-15', 'Latin15/ISO-8859-15'), + ('Big5', 'BIG5'), + ('EUC-JP', 'EUC-JP'), + ('EUC-KR', 'EUC-KR'), + ('GBK', 'GBK'), + ('GB18030', 'GB18030'), + ('Shift_JIS', 'Shift_JIS'), + ('KOI8-R', 'KOI8-R'), + ('KOI8-U', 'KOI8-U'), + ('cp874', 'Windows CP874'), + ('windows-1250', 'Windows CP1250'), + ('windows-1251', 'Windows CP1251'), + ('windows-1252', 'Windows CP1252'), + ('windows-1253', 'Windows CP1253'), + ('windows-1254', 'Windows CP1254'), + ('windows-1255', 'Windows CP1255'), + ('windows-1256', 'Windows CP1256'), + ('windows-1257', 'Windows CP1257'), + ('windows-1258', 'Windows CP1258') +) + +STATE_READY = "READY" +STATE_RUNNING = "RUNNING" +STATE_PENDING = "PENDING" +STATE_WAITING = "WAITING" +STATE_INCOMPLETE = "INCOMPLETE" +STATE_COMPLETE = "COMPLETE" +STATE_INVALID = "INVALID" +STATE_PROCESSED = "PROCESSED" + +PROCESSING_STATES = ( + (STATE_READY, "READY"), + (STATE_RUNNING, "RUNNING"), + (STATE_PENDING, "PENDING"), + (STATE_WAITING, "WAITING"), + (STATE_INCOMPLETE, "INCOMPLETE"), + (STATE_COMPLETE, "COMPLETE"), + (STATE_INVALID, "INVALID"), + (STATE_PROCESSED, "PROCESSED"), +) + +LAYER_TYPES = ['vector', 'raster', 'remote', 'vector_time'] diff --git a/geonode/base/fields.py b/geonode/base/fields.py index 1650df1aa1c..728f23c62c8 100644 --- a/geonode/base/fields.py +++ b/geonode/base/fields.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/base/fixtures/django_celery_beat.json b/geonode/base/fixtures/django_celery_beat.json deleted file mode 100644 index 66c2b6a61fc..00000000000 --- a/geonode/base/fixtures/django_celery_beat.json +++ /dev/null @@ -1,72 +0,0 @@ -[ - { - "model": "django_celery_beat.intervalschedule", - "pk": 1, - "fields": { - "every": 60, - "period": "seconds" - } - }, - { - "model": "django_celery_beat.crontabschedule", - "pk": 1, - "fields": { - "minute": "0", - "hour": "4", - "day_of_week": "*", - "day_of_month": "*", - "month_of_year": "*" - } - }, - { - "model": "django_celery_beat.periodictasks", - "pk": 1, - "fields": { - "last_update": "2019-10-14T12:56:49.352Z" - } - }, - { - "model": "django_celery_beat.periodictask", - "pk": 1, - "fields": { - "name": "celery.backend_cleanup", - "task": "celery.backend_cleanup", - "interval": null, - "crontab": 1, - "solar": null, - "args": "[]", - "kwargs": "{}", - "queue": null, - "exchange": null, - "routing_key": null, - "expires": null, - "enabled": true, - "last_run_at": null, - "total_run_count": 0, - "date_changed": "2019-10-14T12:50:54.847Z", - "description": "" - } - }, - { - "model": "django_celery_beat.periodictask", - "pk": 2, - "fields": { - "name": "delayed-security-sync-task", - "task": "geonode.security.tasks.synch_guardian", - "interval": 1, - "crontab": null, - "solar": null, - "args": "[]", - "kwargs": "{}", - "queue": null, - "exchange": null, - "routing_key": null, - "expires": null, - "enabled": true, - "last_run_at": null, - "total_run_count": 0, - "date_changed": "2019-10-14T12:56:37.554Z", - "description": "" - } - } -] \ No newline at end of file diff --git a/geonode/base/fixtures/test_sld.sld b/geonode/base/fixtures/test_sld.sld index 084d47dde92..7d59c34c30a 100644 --- a/geonode/base/fixtures/test_sld.sld +++ b/geonode/base/fixtures/test_sld.sld @@ -1,29 +1,29 @@ - - - - Simple Point - - SLD Cook Book: Simple Point - - - - - - circle - - #FF0000 - - - 6 - - - - - - - \ No newline at end of file + + + + Simple Point + + SLD Cook Book: Simple Point + + + + + + circle + + #FF0000 + + + 6 + + + + + + + diff --git a/geonode/base/forms.py b/geonode/base/forms.py index 822b8a0e341..9fa8778dfe5 100644 --- a/geonode/base/forms.py +++ b/geonode/base/forms.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -218,7 +217,7 @@ def _region_id_from_choice(choice): else: return choice.id - selected_choices = set(force_text(_region_id_from_choice(v)) for v in selected_choices) + selected_choices = {force_text(_region_id_from_choice(v)) for v in selected_choices} output = [] output.append(format_html('', 'Global')) @@ -316,7 +315,7 @@ class Meta: class ThesaurusAvailableForm(forms.Form): def __init__(self, *args, **kwargs): - super(ThesaurusAvailableForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) lang = get_language() for item in Thesaurus.objects.all().order_by('order', 'id'): tname = self._get_thesauro_title_label(item, lang) @@ -387,7 +386,7 @@ def build_attrs(self, base_attrs=None, extra_attrs=None, **kwargs): if extra_attrs: base_attrs.update(extra_attrs) base_attrs.update(kwargs) - return super(ResourceBaseDateTimePicker, self).build_attrs(base_attrs) + return super().build_attrs(base_attrs) # return base_attrs @@ -397,22 +396,27 @@ class ResourceBaseForm(TranslationModelForm): label=_("Abstract"), required=False, widget=TinyMCE()) + purpose = forms.CharField( label=_("Purpose"), required=False, widget=TinyMCE()) + constraints_other = forms.CharField( label=_("Other constraints"), required=False, widget=TinyMCE()) + supplemental_information = forms.CharField( label=_('Supplemental information'), required=False, widget=TinyMCE()) + data_quality_statement = forms.CharField( label=_("Data quality statement"), required=False, widget=TinyMCE()) + owner = forms.ModelChoiceField( empty_label=_("Owner"), label=_("Owner"), @@ -426,6 +430,7 @@ class ResourceBaseForm(TranslationModelForm): input_formats=['%Y-%m-%d %H:%M %p'], widget=ResourceBaseDateTimePicker(options={"format": "YYYY-MM-DD HH:mm a"}) ) + temporal_extent_start = forms.DateTimeField( label=_("temporal extent start"), required=False, @@ -433,6 +438,7 @@ class ResourceBaseForm(TranslationModelForm): input_formats=['%Y-%m-%d %H:%M %p'], widget=ResourceBaseDateTimePicker(options={"format": "YYYY-MM-DD HH:mm a"}) ) + temporal_extent_end = forms.DateTimeField( label=_("temporal extent end"), required=False, @@ -480,7 +486,7 @@ class ResourceBaseForm(TranslationModelForm): regions.widget.attrs = {"size": 20} def __init__(self, *args, **kwargs): - super(ResourceBaseForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) for field in self.fields: help_text = self.fields[field].help_text if help_text != '': @@ -546,7 +552,10 @@ class Meta: 'tkeywords', 'users_geolimits', 'groups_geolimits', - 'dirty_state' + 'dirty_state', + 'state', + 'blob', + 'files', ) @@ -634,7 +643,7 @@ class BatchPermissionsForm(forms.Form): class UserAndGroupPermissionsForm(forms.Form): def __init__(self, *args, **kwargs): - super(UserAndGroupPermissionsForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.fields['layers'].label_from_instance = self.label_from_instance layers = forms.ModelMultipleChoiceField( diff --git a/geonode/base/management/__init__.py b/geonode/base/management/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/base/management/__init__.py +++ b/geonode/base/management/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/base/management/commands/__init__.py b/geonode/base/management/commands/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/base/management/commands/__init__.py +++ b/geonode/base/management/commands/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/base/management/commands/delete_resources.py b/geonode/base/management/commands/delete_resources.py index 49d29575f86..f3b420ae6dd 100644 --- a/geonode/base/management/commands/delete_resources.py +++ b/geonode/base/management/commands/delete_resources.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo diff --git a/geonode/base/management/commands/dump_thesaurus.py b/geonode/base/management/commands/dump_thesaurus.py index 0b797ee912d..0908a43ee9f 100644 --- a/geonode/base/management/commands/dump_thesaurus.py +++ b/geonode/base/management/commands/dump_thesaurus.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2021 OSGeo diff --git a/geonode/base/management/commands/fixoauthuri.py b/geonode/base/management/commands/fixoauthuri.py index 81a6e57f626..92944ed79db 100644 --- a/geonode/base/management/commands/fixoauthuri.py +++ b/geonode/base/management/commands/fixoauthuri.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/base/management/commands/fixsitename.py b/geonode/base/management/commands/fixsitename.py index 1af5e2ee4ea..3a1caef5a7c 100644 --- a/geonode/base/management/commands/fixsitename.py +++ b/geonode/base/management/commands/fixsitename.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/base/management/commands/helpers.py b/geonode/base/management/commands/helpers.py index fe049229726..226fe55cfea 100644 --- a/geonode/base/management/commands/helpers.py +++ b/geonode/base/management/commands/helpers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo diff --git a/geonode/base/management/commands/lib/gn20_to_24.py b/geonode/base/management/commands/lib/gn20_to_24.py index 19808cf3ba3..db7c27a6761 100644 --- a/geonode/base/management/commands/lib/gn20_to_24.py +++ b/geonode/base/management/commands/lib/gn20_to_24.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -33,7 +32,7 @@ def __init__(self, *args, **kwargs): self.datastore = kwargs.get('datastore', '') self.siteurl = kwargs.get('siteurl', '') - super(DefaultMangler, self).__init__(*args) + super().__init__(*args) def default(self, obj): # Let the base class default method raise the TypeError @@ -43,7 +42,7 @@ def decode(self, json_string): """ json_string is basicly string that you give to json.loads method """ - default_obj = super(DefaultMangler, self).decode(json_string) + default_obj = super().decode(json_string) # manipulate your object any way you want # .... @@ -61,7 +60,7 @@ def decode(self, json_string): """ json_string is basicly string that you give to json.loads method """ - default_obj = super(ResourceBaseMangler, self).decode(json_string) + default_obj = super().decode(json_string) # manipulate your object any way you want # .... @@ -142,7 +141,7 @@ def decode(self, json_string): """ json_string is basicly string that you give to json.loads method """ - default_obj = super(LayerMangler, self).decode(json_string) + default_obj = super().decode(json_string) # manipulate your object any way you want # .... @@ -199,7 +198,7 @@ def decode(self, json_string): """ json_string is basicly string that you give to json.loads method """ - default_obj = super(LayerAttributesMangler, self).decode(json_string) + default_obj = super().decode(json_string) # manipulate your object any way you want # .... @@ -221,7 +220,7 @@ def decode(self, json_string): """ json_string is basicly string that you give to json.loads method """ - default_obj = super(MapMangler, self).decode(json_string) + default_obj = super().decode(json_string) # manipulate your object any way you want # .... @@ -270,7 +269,7 @@ def decode(self, json_string): """ json_string is basicly string that you give to json.loads method """ - default_obj = super(MapLayersMangler, self).decode(json_string) + default_obj = super().decode(json_string) # manipulate your object any way you want # .... diff --git a/geonode/base/management/commands/lib/gn24_to_24.py b/geonode/base/management/commands/lib/gn24_to_24.py index 264805daaf2..223c3023ce3 100644 --- a/geonode/base/management/commands/lib/gn24_to_24.py +++ b/geonode/base/management/commands/lib/gn24_to_24.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -33,7 +32,7 @@ def __init__(self, *args, **kwargs): self.datastore = kwargs.get('datastore', '') self.siteurl = kwargs.get('siteurl', '') - super(DefaultMangler, self).__init__(*args) + super().__init__(*args) def default(self, obj): # Let the base class default method raise the TypeError @@ -43,7 +42,7 @@ def decode(self, json_string): """ json_string is basicly string that you give to json.loads method """ - default_obj = super(DefaultMangler, self).decode(json_string) + default_obj = super().decode(json_string) # manipulate your object any way you want # .... @@ -63,7 +62,7 @@ def decode(self, json_string): """ json_string is basicly string that you give to json.loads method """ - default_obj = super(ResourceBaseMangler, self).decode(json_string) + default_obj = super().decode(json_string) # manipulate your object any way you want # .... @@ -118,7 +117,7 @@ def decode(self, json_string): """ json_string is basicly string that you give to json.loads method """ - default_obj = super(LayerMangler, self).decode(json_string) + default_obj = super().decode(json_string) # manipulate your object any way you want # .... @@ -146,7 +145,7 @@ def decode(self, json_string): """ json_string is basicly string that you give to json.loads method """ - default_obj = super(LayerAttributesMangler, self).decode(json_string) + default_obj = super().decode(json_string) # manipulate your object any way you want # .... @@ -168,7 +167,7 @@ def decode(self, json_string): """ json_string is basicly string that you give to json.loads method """ - default_obj = super(MapLayersMangler, self).decode(json_string) + default_obj = super().decode(json_string) # manipulate your object any way you want # .... diff --git a/geonode/base/management/commands/load_thesaurus.py b/geonode/base/management/commands/load_thesaurus.py index 4f8a7da81b1..c8d51b78ad8 100644 --- a/geonode/base/management/commands/load_thesaurus.py +++ b/geonode/base/management/commands/load_thesaurus.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/base/management/commands/migrate_baseurl.py b/geonode/base/management/commands/migrate_baseurl.py index c6befd74f56..e8c1b628012 100644 --- a/geonode/base/management/commands/migrate_baseurl.py +++ b/geonode/base/management/commands/migrate_baseurl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2017 OSGeo diff --git a/geonode/base/management/commands/migrate_notifications.py b/geonode/base/management/commands/migrate_notifications.py index 19a10a7a861..1bbb6eff469 100644 --- a/geonode/base/management/commands/migrate_notifications.py +++ b/geonode/base/management/commands/migrate_notifications.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2017 OSGeo diff --git a/geonode/base/management/commands/set_all_layers_alternate.py b/geonode/base/management/commands/set_all_layers_alternate.py index 5d2ea1c72a4..9095956bf63 100644 --- a/geonode/base/management/commands/set_all_layers_alternate.py +++ b/geonode/base/management/commands/set_all_layers_alternate.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2017 OSGeo diff --git a/geonode/base/management/commands/set_all_layers_metadata.py b/geonode/base/management/commands/set_all_layers_metadata.py index 7a7014baef5..9afe33f72e7 100644 --- a/geonode/base/management/commands/set_all_layers_metadata.py +++ b/geonode/base/management/commands/set_all_layers_metadata.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2017 OSGeo diff --git a/geonode/base/management/commands/squashmigrations.py b/geonode/base/management/commands/squashmigrations.py index 37122a6b016..26d2131b467 100644 --- a/geonode/base/management/commands/squashmigrations.py +++ b/geonode/base/management/commands/squashmigrations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2017 OSGeo @@ -39,4 +38,4 @@ def handle(self, **options): # Monkeypatch RunPython.noop = noop # Invoke original - return super(Command, self).handle(**options) + return super().handle(**options) diff --git a/geonode/base/middleware.py b/geonode/base/middleware.py index ae71819b665..0c0f918a3cf 100644 --- a/geonode/base/middleware.py +++ b/geonode/base/middleware.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/base/migrations/0027_auto_20170801_1228.py b/geonode/base/migrations/0027_auto_20170801_1228.py index 751593dea1b..1f08a61e0f2 100644 --- a/geonode/base/migrations/0027_auto_20170801_1228.py +++ b/geonode/base/migrations/0027_auto_20170801_1228.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/base/migrations/0027_auto_20170801_1228_squashed_0037_auto_20190222_1347.py b/geonode/base/migrations/0027_auto_20170801_1228_squashed_0037_auto_20190222_1347.py index bd568a3be9e..f0acc20e43f 100644 --- a/geonode/base/migrations/0027_auto_20170801_1228_squashed_0037_auto_20190222_1347.py +++ b/geonode/base/migrations/0027_auto_20170801_1228_squashed_0037_auto_20190222_1347.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-04-04 08:29 @@ -174,11 +173,11 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='menuitem', - unique_together=set([('menu', 'order'), ('menu', 'title')]), + unique_together={('menu', 'order'), ('menu', 'title')}, ), migrations.AlterUniqueTogether( name='menu', - unique_together=set([('placeholder', 'order'), ('placeholder', 'title')]), + unique_together={('placeholder', 'order'), ('placeholder', 'title')}, ), migrations.AlterField( model_name='resourcebase', diff --git a/geonode/base/migrations/0028_curatedthumbnail.py b/geonode/base/migrations/0028_curatedthumbnail.py index 28787c821aa..b894d1eca76 100644 --- a/geonode/base/migrations/0028_curatedthumbnail.py +++ b/geonode/base/migrations/0028_curatedthumbnail.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.23 on 2019-08-14 14:10 diff --git a/geonode/base/migrations/0028_resourcebase_is_approved.py b/geonode/base/migrations/0028_resourcebase_is_approved.py index 2ad3c6e5b12..b0c5445b401 100644 --- a/geonode/base/migrations/0028_resourcebase_is_approved.py +++ b/geonode/base/migrations/0028_resourcebase_is_approved.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/base/migrations/0029_auto_20171114_0341.py b/geonode/base/migrations/0029_auto_20171114_0341.py index f76998b85ec..26dc3aa4a74 100644 --- a/geonode/base/migrations/0029_auto_20171114_0341.py +++ b/geonode/base/migrations/0029_auto_20171114_0341.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/base/migrations/0029_resourcebase_last_updated.py b/geonode/base/migrations/0029_resourcebase_last_updated.py index a277d800ade..1f0e51276e5 100644 --- a/geonode/base/migrations/0029_resourcebase_last_updated.py +++ b/geonode/base/migrations/0029_resourcebase_last_updated.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.25 on 2019-10-04 15:48 diff --git a/geonode/base/migrations/0030_auto_20180309_0833.py b/geonode/base/migrations/0030_auto_20180309_0833.py index 4dd60afa92d..de9d98b4043 100644 --- a/geonode/base/migrations/0030_auto_20180309_0833.py +++ b/geonode/base/migrations/0030_auto_20180309_0833.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2018-03-09 08:33 diff --git a/geonode/base/migrations/0030_resourcebase_created.py b/geonode/base/migrations/0030_resourcebase_created.py index 4a5b16d1cd6..dcb515325f2 100644 --- a/geonode/base/migrations/0030_resourcebase_created.py +++ b/geonode/base/migrations/0030_resourcebase_created.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.25 on 2019-10-04 15:59 diff --git a/geonode/base/migrations/0031_auto_20180309_0837.py b/geonode/base/migrations/0031_auto_20180309_0837.py index 780a3b52f3a..26508197c3b 100644 --- a/geonode/base/migrations/0031_auto_20180309_0837.py +++ b/geonode/base/migrations/0031_auto_20180309_0837.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2018-03-09 08:37 diff --git a/geonode/base/migrations/0031_auto_20200114_1651.py b/geonode/base/migrations/0031_auto_20200114_1651.py index 7b2ce499e78..b13f98a1c18 100644 --- a/geonode/base/migrations/0031_auto_20200114_1651.py +++ b/geonode/base/migrations/0031_auto_20200114_1651.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.27 on 2020-01-14 16:51 diff --git a/geonode/base/migrations/0032_auto_20180329_1844.py b/geonode/base/migrations/0032_auto_20180329_1844.py index d47953553f6..6a4e039e7d2 100644 --- a/geonode/base/migrations/0032_auto_20180329_1844.py +++ b/geonode/base/migrations/0032_auto_20180329_1844.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/base/migrations/0033_auto_20180330_0951.py b/geonode/base/migrations/0033_auto_20180330_0951.py index bf1dad611de..6979bbe9704 100644 --- a/geonode/base/migrations/0033_auto_20180330_0951.py +++ b/geonode/base/migrations/0033_auto_20180330_0951.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/base/migrations/0034_auto_20180606_1543.py b/geonode/base/migrations/0034_auto_20180606_1543.py index 1ec035c87c4..b572a2d9fc9 100644 --- a/geonode/base/migrations/0034_auto_20180606_1543.py +++ b/geonode/base/migrations/0034_auto_20180606_1543.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.11 on 2018-06-06 15:43 diff --git a/geonode/base/migrations/0035_resourcebase_dirty_state.py b/geonode/base/migrations/0035_resourcebase_dirty_state.py index dd568ced92e..ad52f2d6605 100644 --- a/geonode/base/migrations/0035_resourcebase_dirty_state.py +++ b/geonode/base/migrations/0035_resourcebase_dirty_state.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.11 on 2018-10-02 15:42 diff --git a/geonode/base/migrations/0036_auto_20190129_1433.py b/geonode/base/migrations/0036_auto_20190129_1433.py index e8089b28ba0..afe3745b882 100644 --- a/geonode/base/migrations/0036_auto_20190129_1433.py +++ b/geonode/base/migrations/0036_auto_20190129_1433.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2019-01-29 14:33 @@ -52,10 +51,10 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='menuitem', - unique_together=set([('menu', 'order'), ('menu', 'title')]), + unique_together={('menu', 'order'), ('menu', 'title')}, ), migrations.AlterUniqueTogether( name='menu', - unique_together=set([('placeholder', 'order'), ('placeholder', 'title')]), + unique_together={('placeholder', 'order'), ('placeholder', 'title')}, ), ] diff --git a/geonode/base/migrations/0037_auto_20190222_1347.py b/geonode/base/migrations/0037_auto_20190222_1347.py index c48edde8ed5..00ddf66f81c 100644 --- a/geonode/base/migrations/0037_auto_20190222_1347.py +++ b/geonode/base/migrations/0037_auto_20190222_1347.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-02-22 13:47 diff --git a/geonode/base/migrations/0060_resourcebase_state.py b/geonode/base/migrations/0060_resourcebase_state.py new file mode 100644 index 00000000000..06900c9aba9 --- /dev/null +++ b/geonode/base/migrations/0060_resourcebase_state.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.20 on 2021-05-25 09:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0059_fill_empty_resource_type'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='state', + field=models.CharField(choices=[('READY', 'READY'), ('RUNNING', 'RUNNING'), ('PENDING', 'PENDING'), ('WAITING', 'WAITING'), ('INCOMPLETE', 'INCOMPLETE'), ('COMPLETE', 'COMPLETE'), ('INVALID', 'INVALID'), ('PROCESSED', 'PROCESSED')], default='READY', help_text='Hold the resource processing state.', max_length=16, verbose_name='State'), + ), + ] diff --git a/geonode/base/migrations/0061_merge_0060_auto_20210512_1658_0060_resourcebase_state.py b/geonode/base/migrations/0061_merge_0060_auto_20210512_1658_0060_resourcebase_state.py new file mode 100644 index 00000000000..248ee135478 --- /dev/null +++ b/geonode/base/migrations/0061_merge_0060_auto_20210512_1658_0060_resourcebase_state.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2 on 2021-05-25 15:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0060_auto_20210512_1658'), + ('base', '0060_resourcebase_state'), + ] + + operations = [ + ] diff --git a/geonode/base/migrations/0062_resourcebase_files.py b/geonode/base/migrations/0062_resourcebase_files.py new file mode 100644 index 00000000000..07db217d3a3 --- /dev/null +++ b/geonode/base/migrations/0062_resourcebase_files.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2 on 2021-05-28 15:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0061_merge_0060_auto_20210512_1658_0060_resourcebase_state'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='files', + field=models.JSONField(default=dict), + ), + ] diff --git a/geonode/base/migrations/0063_alter_resourcebase_files.py b/geonode/base/migrations/0063_alter_resourcebase_files.py new file mode 100644 index 00000000000..ff0dfaea5df --- /dev/null +++ b/geonode/base/migrations/0063_alter_resourcebase_files.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2 on 2021-06-03 16:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0062_resourcebase_files'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='files', + field=models.JSONField(default=dict, null=True), + ), + ] diff --git a/geonode/base/migrations/0064_alter_resourcebase_files.py b/geonode/base/migrations/0064_alter_resourcebase_files.py new file mode 100644 index 00000000000..2e74619b27b --- /dev/null +++ b/geonode/base/migrations/0064_alter_resourcebase_files.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2 on 2021-06-04 07:42 + +from django.db import migrations, models +from geonode.base.models import ResourceBase + +def change_value(apps, schema_editor): + resources = apps.get_model('base', 'ResourceBase') + for r in resources.objects.all(): + if isinstance(r.files, dict): + r2 = ResourceBase.objects.filter(id=r.id) + f = list(r.files.values()) + r2.update(**{'files': f}) + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0063_alter_resourcebase_files'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='files', + field=models.JSONField(blank=True, default=list, null=True), + ), + migrations.RunPython(change_value), + + ] diff --git a/geonode/base/migrations/0065_alter_curatedthumbnail_img.py b/geonode/base/migrations/0065_alter_curatedthumbnail_img.py new file mode 100644 index 00000000000..9426304d65e --- /dev/null +++ b/geonode/base/migrations/0065_alter_curatedthumbnail_img.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.4 on 2021-06-09 13:58 + +from django.db import migrations, models +import geonode.storage.manager + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0064_alter_resourcebase_files'), + ] + + operations = [ + migrations.AlterField( + model_name='curatedthumbnail', + name='img', + field=models.ImageField(storage=geonode.storage.manager.StorageManager(), upload_to='curated_thumbs'), + ), + ] diff --git a/geonode/base/migrations/0066_resourcebase_data.py b/geonode/base/migrations/0066_resourcebase_data.py new file mode 100644 index 00000000000..87a6989bef9 --- /dev/null +++ b/geonode/base/migrations/0066_resourcebase_data.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2 on 2021-06-11 13:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0065_alter_curatedthumbnail_img'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='blob', + field=models.JSONField(blank=True, default=dict, null=True), + ), + ] diff --git a/geonode/base/migrations/0067_resourcebase_storetype.py b/geonode/base/migrations/0067_resourcebase_storetype.py new file mode 100644 index 00000000000..f6f9a5c1ea7 --- /dev/null +++ b/geonode/base/migrations/0067_resourcebase_storetype.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.4 on 2021-07-06 08:48 + +from django.db import migrations, models +from django.db.migrations import RunSQL + + +clone_layers_storetypes = ''' +UPDATE base_resourcebase +SET "storetype"=subquery."storeType" +FROM (select resourcebase_ptr_id,"storeType" from layers_layer gg) AS subquery +WHERE base_resourcebase.id=subquery.resourcebase_ptr_id; +''' + +clone_documents_storetypes = ''' +UPDATE base_resourcebase +SET "storetype"=subquery."doc_type" +FROM (select resourcebase_ptr_id,"doc_type" from documents_document gg) AS subquery +WHERE base_resourcebase.id=subquery.resourcebase_ptr_id; +''' + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0032_remove_document_doc_file'), + ('layers', '0035_auto_20210525_0847'), + ('base', '0066_resourcebase_data'), + ] + + operations = [ + migrations.AddField( + model_name='resourcebase', + name='storetype', + field=models.CharField(blank=True, max_length=128, null=True), + ), + RunSQL(clone_layers_storetypes), + RunSQL(clone_documents_storetypes) + ] diff --git a/geonode/base/migrations/24_initial.py b/geonode/base/migrations/24_initial.py index 87fde261d90..a18839e0dce 100644 --- a/geonode/base/migrations/24_initial.py +++ b/geonode/base/migrations/24_initial.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models import mptt.fields import geonode.security.models @@ -224,6 +221,6 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='contactrole', - unique_together=set([('contact', 'resource', 'role')]), + unique_together={('contact', 'resource', 'role')}, ), ] diff --git a/geonode/base/migrations/24_to_26.py b/geonode/base/migrations/24_to_26.py index afcf06b11f5..df7d89b240e 100644 --- a/geonode/base/migrations/24_to_26.py +++ b/geonode/base/migrations/24_to_26.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models import taggit.managers @@ -118,10 +115,10 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='thesauruskeywordlabel', - unique_together=set([('keyword', 'lang')]), + unique_together={('keyword', 'lang')}, ), migrations.AlterUniqueTogether( name='thesauruskeyword', - unique_together=set([('thesaurus', 'alt_label')]), + unique_together={('thesaurus', 'alt_label')}, ), ] diff --git a/geonode/base/migrations/26_to_27.py b/geonode/base/migrations/26_to_27.py index d8acd62daa2..dd60a652829 100644 --- a/geonode/base/migrations/26_to_27.py +++ b/geonode/base/migrations/26_to_27.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/base/models.py b/geonode/base/models.py index a4a29669766..c577adb063f 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -22,27 +21,30 @@ import re import html import math -import uuid import logging import traceback +from sequences.models import Sequence + +from sequences import get_next_value from django.db import models +from django.db.models import Max from django.conf import settings from django.core import serializers -from django.utils.functional import cached_property from django.utils.html import escape from django.utils.timezone import now from django.db.models import Q, signals from django.contrib.auth.models import Group from django.core.files.base import ContentFile from django.contrib.auth import get_user_model -from django.contrib.gis.geos import GEOSGeometry, Polygon, Point +from django.db.models.fields.json import JSONField +from django.utils.functional import cached_property +from django.contrib.gis.geos import Polygon, Point from django.contrib.gis.db.models import PolygonField -from django.core.exceptions import ValidationError +from django.core.exceptions import SuspiciousFileOperation, ValidationError from django.utils.translation import ugettext_lazy as _ from django.contrib.contenttypes.models import ContentType from django.templatetags.static import static -from django.core.files.storage import default_storage as storage from django.utils.html import strip_tags from mptt.models import MPTTModel, TreeForeignKey @@ -61,18 +63,12 @@ from guardian.shortcuts import get_anonymous_user, get_objects_for_user from treebeard.mp_tree import MP_Node, MP_NodeQuerySet, MP_NodeManager +from geonode.base import enumerations from geonode.singleton import SingletonModel -from geonode.base.enumerations import ( - LINK_TYPES, - ALL_LANGUAGES, - HIERARCHY_LEVELS, - UPDATE_FREQUENCIES, - DEFAULT_SUPPLEMENTAL_INFORMATION) from geonode.base.bbox_utils import BBOXHelper, polygon_from_bbox from geonode.utils import ( - is_monochromatic_image, - add_url_params, - bbox_to_wkt) + bbox_to_wkt, + is_monochromatic_image) from geonode.groups.models import GroupProfile from geonode.security.utils import get_visible_resources from geonode.security.models import PermissionLevelMixin @@ -88,8 +84,9 @@ from pyproj import transform, Proj -from urllib.parse import urlparse, urlsplit, urljoin +from urllib.parse import urlsplit, urljoin from imagekit.cachefiles.backends import Simple +from geonode.storage.manager import storage_manager logger = logging.getLogger(__name__) @@ -326,7 +323,7 @@ class HierarchicalKeywordQuerySet(MP_NodeQuerySet): def create(self, **kwargs): if 'depth' not in kwargs: return self.model.add_root(**kwargs) - return super(HierarchicalKeywordQuerySet, self).create(**kwargs) + return super().create(**kwargs) class HierarchicalKeywordManager(MP_NodeManager): @@ -427,11 +424,11 @@ def tags_for(cls, model, instance=None): class _HierarchicalTagManager(_TaggableManager): def add(self, *tags): - str_tags = set([ + str_tags = { t for t in tags if not isinstance(t, self.through.tag_model()) - ]) + } tag_objs = set(tags) - str_tags # If str_tags has 0 elements Django actually optimizes that to not do a # query. Malcolm is very smart. @@ -439,7 +436,7 @@ def add(self, *tags): name__in=str_tags ) tag_objs.update(existing) - for new_tag in str_tags - set(t.name for t in existing): + for new_tag in str_tags - {t.name for t in existing}: if new_tag: new_tag = escape(new_tag) tag_objs.add(HierarchicalKeyword.add_root(name=new_tag)) @@ -588,12 +585,30 @@ def admin_contact(self): return superusers[0] def get_queryset(self): - return super( - ResourceBaseManager, - self).get_queryset().non_polymorphic() + return super().get_queryset().non_polymorphic() def polymorphic_queryset(self): - return super(ResourceBaseManager, self).get_queryset() + return super().get_queryset() + + @staticmethod + def upload_files(resource_id, files): + try: + out = [] + for f in files: + if os.path.isfile(f) and os.path.exists(f): + + with open(f, 'rb') as ff: + folder = os.path.basename(os.path.dirname(f)) + filename = os.path.basename(f) + file_uploaded_path = storage_manager.save(f'{folder}/{filename}', ff) + out.append(storage_manager.path(file_uploaded_path)) + + # making an update instead of save in order to avoid others + # signal like post_save and commiunication with geoserver + ResourceBase.objects.filter(id=resource_id).update(files=out) + return out + except Exception as e: + logger.exception(e) class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): @@ -714,7 +729,7 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): maintenance_frequency = models.CharField( _('maintenance frequency'), max_length=255, - choices=UPDATE_FREQUENCIES, + choices=enumerations.UPDATE_FREQUENCIES, blank=True, null=True, help_text=maintenance_frequency_help_text) @@ -759,7 +774,7 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): language = models.CharField( _('language'), max_length=3, - choices=ALL_LANGUAGES, + choices=enumerations.ALL_LANGUAGES, default='eng', help_text=language_help_text) category = models.ForeignKey( @@ -792,7 +807,7 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): supplemental_information = models.TextField( _('supplemental information'), max_length=2000, - default=DEFAULT_SUPPLEMENTAL_INFORMATION, + default=enumerations.DEFAULT_SUPPLEMENTAL_INFORMATION, help_text=_('any other descriptive information about the dataset')) # Section 8 @@ -846,7 +861,7 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): max_length=32, default='dataset', null=False, - choices=HIERARCHY_LEVELS) + choices=enumerations.HIERARCHY_LEVELS) csw_anytext = models.TextField(_('CSW anytext'), null=True, blank=True) csw_wkt_geometry = models.TextField( _('CSW WKT geometry'), @@ -862,8 +877,10 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): blank=True) popular_count = models.IntegerField(default=0) share_count = models.IntegerField(default=0) - featured = models.BooleanField(_("Featured"), default=False, help_text=_( - 'Should this resource be advertised in home page?')) + featured = models.BooleanField( + _("Featured"), + default=False, + help_text=_('Should this resource be advertised in home page?')) is_published = models.BooleanField( _("Is Published"), default=True, @@ -880,6 +897,15 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): created = models.DateTimeField(auto_now_add=True, null=True, blank=True) last_updated = models.DateTimeField(auto_now=True, null=True, blank=True) + state = models.CharField( + _("State"), + max_length=16, + null=False, + blank=False, + default=enumerations.STATE_READY, + choices=enumerations.PROCESSING_STATES, + help_text=_('Hold the resource processing state.')) + # fields controlling security state dirty_state = models.BooleanField( _("Dirty State"), @@ -909,6 +935,12 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): default=False, help_text=_('If true, will be excluded from search')) + files = JSONField(null=True, default=list, blank=True) + + blob = JSONField(null=True, default=dict, blank=True) + + storetype = models.CharField(max_length=128, null=True, blank=True) + __is_approved = False __is_published = False @@ -931,7 +963,7 @@ def __init__(self, *args, **kwargs): if all(bbox): kwargs['bbox_polygon'] = Polygon.from_bbox(bbox) kwargs['ll_bbox_polygon'] = Polygon.from_bbox(bbox) - super(ResourceBase, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def __str__(self): return str(self.title) @@ -977,9 +1009,10 @@ def save(self, notify=False, *args, **kwargs): self.resource_type = self.polymorphic_ctype.model.lower() if hasattr(self, 'class_name') and (self.pk is None or notify): - if self.pk is None and self.title: + if self.pk is None and (self.title or getattr(self, 'name', None)): # Resource Created - + if not self.title and getattr(self, 'name', None): + self.title = getattr(self, 'name', None) notice_type_label = f'{self.class_name.lower()}_created' recipients = get_notification_recipients(notice_type_label, resource=self) send_notification(recipients, notice_type_label, {'resource': self}) @@ -1017,7 +1050,22 @@ def save(self, notify=False, *args, **kwargs): recipients = get_notification_recipients(notice_type_label, resource=self) send_notification(recipients, notice_type_label, {'resource': self}) - super(ResourceBase, self).save(*args, **kwargs) + if self.pk is None: + _initial_value = ResourceBase.objects.aggregate(Max("pk"))['pk__max'] + if not _initial_value: + _initial_value = 1 + else: + _initial_value += 1 + _next_value = get_next_value( + "ResourceBase", # type(self).__name__, + initial_value=_initial_value) + if _initial_value > _next_value: + Sequence.objects.filter(name='ResourceBase').update(last=_initial_value) + _next_value = _initial_value + + self.pk = self.id = _next_value + + super().save(*args, **kwargs) self.__is_approved = self.is_approved self.__is_published = self.is_published @@ -1025,12 +1073,15 @@ def delete(self, notify=True, *args, **kwargs): """ Send a notification when a layer, map or document is deleted """ + from geonode.resource.manager import resource_manager + resource_manager.remove_permissions(self.uuid, instance=self.get_real_instance()) + if hasattr(self, 'class_name') and notify: notice_type_label = f'{self.class_name.lower()}_deleted' recipients = get_notification_recipients(notice_type_label, resource=self) send_notification(recipients, notice_type_label, {'resource': self}) - super(ResourceBase, self).delete(*args, **kwargs) + super().delete(*args, **kwargs) def get_upload_session(self): raise NotImplementedError() @@ -1243,8 +1294,8 @@ def spatial_representation_type_string(self): if hasattr(self.spatial_representation_type, 'identifier'): return self.spatial_representation_type.identifier else: - if hasattr(self, 'storeType'): - if self.storeType == 'coverageStore': + if hasattr(self, 'storetype'): + if self.storetype == 'raster': return 'grid' return 'vector' else: @@ -1258,6 +1309,14 @@ def clear_dirty_state(self): self.dirty_state = False ResourceBase.objects.filter(id=self.id).update(dirty_state=False) + def set_processing_state(self, state): + self.state = state + ResourceBase.objects.filter(id=self.id).update(state=state) + if state == enumerations.STATE_PROCESSED: + self.clear_dirty_state() + else: + self.set_dirty_state() + @property def processed(self): return not self.dirty_state @@ -1402,8 +1461,8 @@ def download_links(self): else: _link_type = 'WWW:DOWNLOAD-1.0-http--download' try: - _store_type = getattr(self.get_real_instance(), 'storeType', None) - if _store_type and _store_type == 'remoteStore' and link.extension in ('html'): + _store_type = getattr(self.get_real_instance(), 'storetype', None) + if _store_type and _store_type in ['tileStore', 'remote'] and link.extension in ('html'): _remote_service = getattr(self.get_real_instance(), '_remote_service', None) if _remote_service: _link_type = f'WWW:DOWNLOAD-{_remote_service.type}' @@ -1486,12 +1545,10 @@ def get_thumbnail_url(self): _thumbnail_url = self.thumbnail_url or static(settings.MISSING_THUMBNAIL) local_thumbnails = self.link_set.filter(name='Thumbnail') remote_thumbnails = self.link_set.filter(name='Remote Thumbnail') - if local_thumbnails.count() > 0: - _thumbnail_url = add_url_params( - local_thumbnails[0].url, {'v': str(uuid.uuid4())[:8]}) - elif remote_thumbnails.count() > 0: - _thumbnail_url = add_url_params( - remote_thumbnails[0].url, {'v': str(uuid.uuid4())[:8]}) + if local_thumbnails.exists(): + _thumbnail_url = local_thumbnails.first().url + elif remote_thumbnails.exists(): + _thumbnail_url = remote_thumbnails.first().url return _thumbnail_url def has_thumbnail(self): @@ -1502,7 +1559,6 @@ def has_thumbnail(self): # that indexing (or other listeners) are notified def save_thumbnail(self, filename, image): upload_path = thumb_path(filename) - try: # Check that the image is valid if is_monochromatic_image(None, image): @@ -1515,31 +1571,30 @@ def save_thumbnail(self, filename, image): if upload_path and image: name, ext = os.path.splitext(filename) remove_thumbs(name) - actual_name = storage.save(upload_path, ContentFile(image)) - url = storage.url(actual_name) - _url = urlparse(url) - _upload_path = thumb_path(os.path.basename(_url.path)) - if upload_path != _upload_path: - if storage.exists(_upload_path): - storage.delete(_upload_path) - try: - os.rename( - storage.path(upload_path), - storage.path(_upload_path) - ) - except Exception as e: - logger.exception(e) + actual_name = storage_manager.save(upload_path, ContentFile(image)) + url = storage_manager.url(actual_name) try: # Optimize the Thumbnail size and resolution _default_thumb_size = getattr( settings, 'THUMBNAIL_GENERATOR_DEFAULT_SIZE', {'width': 240, 'height': 200}) - im = Image.open(open(storage.path(_upload_path), mode='rb')) + im = Image.open(storage_manager.open(actual_name)) im.thumbnail( (_default_thumb_size['width'], _default_thumb_size['height']), resample=Image.ANTIALIAS) cover = ImageOps.fit(im, (_default_thumb_size['width'], _default_thumb_size['height'])) - cover.save(storage.path(_upload_path), format='PNG') + + # Saving the thumb into a temporary directory on file system + tmp_location = f"{settings.MEDIA_ROOT}/{upload_path}" + cover.save(tmp_location, format='PNG') + + with open(tmp_location, 'rb+') as img: + # Saving the img via storage manager + storage_manager.save(storage_manager.path(upload_path), img) + + # If we use a remote storage, the local img is deleted + if tmp_location != storage_manager.path(upload_path): + os.remove(tmp_location) except Exception as e: logger.exception(e) @@ -1550,7 +1605,7 @@ def save_thumbnail(self, filename, image): site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL url = urljoin(site_url, url) - if thumb_size(_upload_path) == 0: + if thumb_size(upload_path) == 0: raise Exception("Generated thumbnail image is zero size") # should only have one 'Thumbnail' link @@ -1635,10 +1690,10 @@ def set_missing_info(self): self.set_default_permissions(owner=user) def maintenance_frequency_title(self): - return [v for v in UPDATE_FREQUENCIES if v[0] == self.maintenance_frequency][0][1].title() + return [v for v in enumerations.UPDATE_FREQUENCIES if v[0] == self.maintenance_frequency][0][1].title() def language_title(self): - return [v for v in ALL_LANGUAGES if v[0] == self.language][0][1].title() + return [v for v in enumerations.ALL_LANGUAGES if v[0] == self.language][0][1].title() def _set_poc(self, poc): # reset any poc assignation to this resource @@ -1745,7 +1800,7 @@ class Link(models.Model): help_text=_('For example "kml"')) link_type = models.CharField( max_length=255, choices=[ - (x, x) for x in LINK_TYPES]) + (x, x) for x in enumerations.LINK_TYPES]) name = models.CharField(max_length=255, help_text=_( 'For example "View in Google Earth"')) mime = models.CharField(max_length=255, @@ -1851,9 +1906,10 @@ class Meta: class CuratedThumbnail(models.Model): resource = models.OneToOneField(ResourceBase, on_delete=models.CASCADE) - img = models.ImageField(upload_to='curated_thumbs') + img = models.ImageField(upload_to='curated_thumbs', storage=storage_manager) # TOD read thumb size from settings img_thumbnail = ImageSpecField(source='img', + cachefile_storage=storage_manager, processors=[ResizeToFill(240, 180)], format='PNG', options={'quality': 60}) @@ -1863,15 +1919,15 @@ def thumbnail_url(self): try: if not Simple()._exists(self.img_thumbnail): Simple().generate(self.img_thumbnail, force=True) - upload_path = storage.path(self.img_thumbnail.name) - actual_name = os.path.basename(storage.url(upload_path)) - _upload_path = os.path.join(os.path.dirname(upload_path), actual_name) - if not os.path.exists(_upload_path): - os.rename(upload_path, _upload_path) - return self.img_thumbnail.url + except SuspiciousFileOperation: + ''' + we must rely to the storage_manager, if the storage is changed, we will ignore this + ''' + return '' except Exception as e: logger.exception(e) - return '' + + return self.img_thumbnail.url or '' class Configuration(SingletonModel): @@ -1925,75 +1981,6 @@ class GroupGeoLimit(models.Model): blank=True) -def resourcebase_post_save(instance, *args, **kwargs): - """ - Used to fill any additional fields after the save. - Has to be called by the children - """ - try: - # set default License if no specified - if instance.license is None: - license = License.objects.filter(name="Not Specified") - - if license and len(license) > 0: - instance.license = license[0] - - ResourceBase.objects.filter(id=instance.id).update( - thumbnail_url=instance.get_thumbnail_url(), - detail_url=instance.get_absolute_url(), - csw_insert_date=now(), - license=instance.license) - instance.refresh_from_db() - except Exception: - tb = traceback.format_exc() - if tb: - logger.debug(tb) - finally: - instance.set_missing_info() - - try: - if not instance.regions or instance.regions.count() == 0: - srid1, wkt1 = instance.geographic_bounding_box.split(";") - srid1 = re.findall(r'\d+', srid1) - - poly1 = GEOSGeometry(wkt1, srid=int(srid1[0])) - poly1.transform(4326) - - queryset = Region.objects.all().order_by('name') - global_regions = [] - regions_to_add = [] - for region in queryset: - try: - srid2, wkt2 = region.geographic_bounding_box.split(";") - srid2 = re.findall(r'\d+', srid2) - - poly2 = GEOSGeometry(wkt2, srid=int(srid2[0])) - poly2.transform(4326) - - if poly2.intersection(poly1): - regions_to_add.append(region) - if region.level == 0 and region.parent is None: - global_regions.append(region) - except Exception: - tb = traceback.format_exc() - if tb: - logger.debug(tb) - if regions_to_add or global_regions: - if regions_to_add and len( - regions_to_add) > 0 and len(regions_to_add) <= 30: - instance.regions.add(*regions_to_add) - else: - instance.regions.add(*global_regions) - except Exception: - tb = traceback.format_exc() - if tb: - logger.debug(tb) - finally: - # refresh catalogue metadata records - from geonode.catalogue.models import catalogue_post_save - catalogue_post_save(instance=instance, sender=instance.__class__) - - def rating_post_save(instance, *args, **kwargs): """ Used to fill the average rating field on OverallRating change. diff --git a/geonode/base/populate_test_data.py b/geonode/base/populate_test_data.py index 5b4ce63ca35..d7ca4ed6fed 100644 --- a/geonode/base/populate_test_data.py +++ b/geonode/base/populate_test_data.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -27,20 +26,21 @@ from taggit.models import TaggedItem from datetime import datetime, timedelta +from django.conf import settings from django.db import transaction from django.utils import timezone from django.contrib.gis.geos import Polygon -from django.contrib.auth.models import Permission, Group from django.core.serializers import serialize from django.contrib.auth import get_user_model +from django.contrib.auth.models import Permission, Group from django.core.files.uploadedfile import SimpleUploadedFile -from geonode import geoserver # noqa from geonode.maps.models import Map +from geonode.base import enumerations from geonode.layers.models import Layer from geonode.compat import ensure_string -from geonode.base.models import ResourceBase, TopicCategory from geonode.documents.models import Document +from geonode.base.models import ResourceBase, TopicCategory # This is used to populate the database with the search fixture data. This is # primarily used as a first step to generate the json data for the fixture using @@ -53,6 +53,7 @@ b'\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;' ) f = SimpleUploadedFile('test_img_file.gif', imgfile.read(), 'image/gif') +dfile = [f"{settings.MEDIA_ROOT}/img.gif"] def all_public(): @@ -60,12 +61,15 @@ def all_public(): for lyr in Layer.objects.all(): lyr.set_default_permissions() lyr.clear_dirty_state() + lyr.set_processing_state(enumerations.STATE_PROCESSED) for mp in Map.objects.all(): mp.set_default_permissions() mp.clear_dirty_state() + mp.set_processing_state(enumerations.STATE_PROCESSED) for doc in Document.objects.all(): doc.set_default_permissions() doc.clear_dirty_state() + doc.set_processing_state(enumerations.STATE_PROCESSED) ResourceBase.objects.all().update(dirty_state=False) @@ -206,6 +210,7 @@ def create_models(type=None, integration=False): m.save() m.set_default_permissions() m.clear_dirty_state() + m.set_processing_state(enumerations.STATE_PROCESSED) obj_ids.append(m.id) for kw in kws: m.keywords.add(kw) @@ -222,20 +227,20 @@ def create_models(type=None, integration=False): bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid='EPSG:4326', - category=category, - doc_file=f, + files=dfile, metadata_only=title == 'doc metadata true' ) m.save() m.set_default_permissions() m.clear_dirty_state() + m.set_processing_state(enumerations.STATE_PROCESSED) obj_ids.append(m.id) for kw in kws: m.keywords.add(kw) m.save() if not type or ensure_string(type) == 'layer': - for ld, owner, storeType in zip(layer_data, cycle(users), cycle(('coverageStore', 'dataStore'))): + for ld, owner, storetype in zip(layer_data, cycle(users), cycle(('raster', 'vector'))): title, abstract, name, alternate, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), start, kws, category = ld end = start + timedelta(days=365) logger.debug(f"[SetUp] Add layer {title}") @@ -252,13 +257,14 @@ def create_models(type=None, integration=False): temporal_extent_start=start, temporal_extent_end=end, date=start, - storeType=storeType, + storetype=storetype, category=category, metadata_only=title == 'layer metadata true' ) layer.save() layer.set_default_permissions() layer.clear_dirty_state() + layer.set_processing_state(enumerations.STATE_PROCESSED) obj_ids.append(layer.id) for kw in kws: layer.keywords.add(kw) @@ -340,13 +346,14 @@ def create_single_layer(name): temporal_extent_start=test_datetime, temporal_extent_end=test_datetime, date=start, - storeType="dataStore", + storetype="vector", resource_type="layer", typename=f"geonode:{title}" ) layer.save() layer.set_default_permissions() layer.clear_dirty_state() + layer.set_processing_state(enumerations.STATE_PROCESSED) return layer @@ -378,6 +385,7 @@ def create_single_map(name): m.save() m.set_default_permissions() m.clear_dirty_state() + m.set_processing_state(enumerations.STATE_PROCESSED) return m @@ -401,12 +409,13 @@ def create_single_doc(name): bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid='EPSG:4326', - doc_file=f, + files=dfile, resource_type="document" ) m.save() m.set_default_permissions() m.clear_dirty_state() + m.set_processing_state(enumerations.STATE_PROCESSED) return m diff --git a/geonode/base/templates/base/_resourcebase_snippet.html b/geonode/base/templates/base/_resourcebase_snippet.html index 490caa207b2..dffe6e9e10c 100644 --- a/geonode/base/templates/base/_resourcebase_snippet.html +++ b/geonode/base/templates/base/_resourcebase_snippet.html @@ -28,9 +28,9 @@

- - - + + + @@ -53,7 +53,7 @@

{% endif %} - + {% trans "Service is" %} {% trans "online" %} {% trans "Service is" %} {% trans "offline" %} diff --git a/geonode/base/templatetags/__init__.py b/geonode/base/templatetags/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/base/templatetags/__init__.py +++ b/geonode/base/templatetags/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/base/templatetags/base_tags.py b/geonode/base/templatetags/base_tags.py index 58d94871b88..370d43147db 100644 --- a/geonode/base/templatetags/base_tags.py +++ b/geonode/base/templatetags/base_tags.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -178,8 +177,8 @@ def facets(context): if not settings.SKIP_PERMS_FILTER: documents = documents.filter(id__in=authorized) - counts = documents.values('doc_type').annotate(count=Count('doc_type')) - facets = {count['doc_type']: count['count'] for count in counts} + counts = documents.values('storetype').annotate(count=Count('storetype')) + facets = {count['storetype']: count['count'] for count in counts} return facets else: @@ -227,28 +226,28 @@ def facets(context): if not settings.SKIP_PERMS_FILTER: layers = layers.filter(id__in=authorized) - counts = layers.values('storeType').annotate(count=Count('storeType')) + counts = layers.values('storetype').annotate(count=Count('storetype')) counts_array = [] try: for count in counts: - counts_array.append((count['storeType'], count['count'])) + counts_array.append((count['storetype'], count['count'])) except Exception: pass count_dict = dict(counts_array) - vector_time_series = layers.exclude(has_time=False).filter(storeType='dataStore'). \ - values('storeType').annotate(count=Count('storeType')) + vector_time_series = layers.exclude(has_time=False).filter(storetype='vector'). \ + values('storetype').annotate(count=Count('storetype')) if vector_time_series: count_dict['vectorTimeSeries'] = vector_time_series[0]['count'] facets = { - 'raster': count_dict.get('coverageStore', 0), - 'vector': count_dict.get('dataStore', 0), + 'raster': count_dict.get('raster', 0), + 'vector': count_dict.get('vector', 0), 'vector_time': count_dict.get('vectorTimeSeries', 0), - 'remote': count_dict.get('remoteStore', 0), + 'remote': count_dict.get('remote', 0), 'wms': count_dict.get('wmsStore', 0), } @@ -404,7 +403,7 @@ def _has_owner_his_permissions(): resource.BASE_PERMISSIONS.get('download')) - \ set(perms) return _owner_set == set() or \ - _owner_set == set(['change_resourcebase_permissions', 'publish_resourcebase']) + _owner_set == {'change_resourcebase_permissions', 'publish_resourcebase'} if not _has_owner_his_permissions() and \ (user.is_superuser or resource.owner.pk == user.pk): diff --git a/geonode/base/templatetags/user_messages.py b/geonode/base/templatetags/user_messages.py index ce965a2dbc1..f771de34eac 100644 --- a/geonode/base/templatetags/user_messages.py +++ b/geonode/base/templatetags/user_messages.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/base/tests.py b/geonode/base/tests.py index 8f94025dc39..e8147888dac 100644 --- a/geonode/base/tests.py +++ b/geonode/base/tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -19,25 +18,26 @@ ######################################################################### import os -from unittest.mock import patch, Mock -from urllib.parse import urlparse - import requests + +from urllib.parse import urlparse +from unittest.mock import patch, Mock from django.core.exceptions import ObjectDoesNotExist -from guardian.shortcuts import assign_perm, get_perms -from imagekit.cachefiles.backends import Simple from io import BytesIO from PIL import Image +from imagekit.cachefiles.backends import Simple +from guardian.shortcuts import assign_perm, get_perms -from geonode.base.utils import OwnerRightsRequestViewUtils, ManageResourceOwnerPermissions -from geonode.base.templatetags.base_tags import display_change_perms_button -from geonode.documents.models import Document -from geonode.layers.models import Layer from geonode.maps.models import Map +from geonode.base import thumb_utils +from geonode.base import enumerations +from geonode.layers.models import Layer from geonode.services.models import Service +from geonode.documents.models import Document from geonode.tests.base import GeoNodeBaseTestSupport -from geonode.base import thumb_utils +from geonode.base.templatetags.base_tags import display_change_perms_button +from geonode.base.utils import OwnerRightsRequestViewUtils, ManageResourceOwnerPermissions from geonode.base.models import ( ResourceBase, MenuPlaceholder, @@ -52,7 +52,7 @@ from django.conf import settings from django.template import Template, Context from django.contrib.auth import get_user_model -from django.core.files.storage import default_storage as storage +from geonode.storage.manager import storage_manager from django.test import Client, TestCase, override_settings, SimpleTestCase from django.shortcuts import reverse @@ -80,7 +80,7 @@ class ThumbnailTests(GeoNodeBaseTestSupport): def setUp(self): - super(ThumbnailTests, self).setUp() + super().setUp() self.rb = ResourceBase.objects.create(owner=get_user_model().objects.get(username='admin')) def tearDown(self): @@ -129,7 +129,7 @@ def test_thumb_utils_methods(self, image): self.assertFalse(thumb_utils.thumb_exists(filename)) f = BytesIO(test_image.tobytes()) f.name = filename - storage.save(upload_path, File(f)) + storage_manager.save(upload_path, File(f)) self.assertTrue(thumb_utils.thumb_exists(filename)) self.assertEqual(thumb_utils.thumb_size(upload_path), 10000) @@ -141,7 +141,7 @@ def test_thumb_utils_methods(self, image): class TestThumbnailUrl(GeoNodeBaseTestSupport): def setUp(self): - super(TestThumbnailUrl, self).setUp() + super().setUp() rb = ResourceBase.objects.create(owner=get_user_model().objects.get(username='admin')) f = BytesIO(test_image.tobytes()) f.name = 'test_image.jpeg' @@ -183,7 +183,7 @@ class RenderMenuTagTest(GeoNodeBaseTestSupport): """ def setUp(self): - super(RenderMenuTagTest, self).setUp() + super().setUp() self.placeholder_0 = MenuPlaceholder.objects.create( name='test_menu_placeholder_0' ) @@ -603,7 +603,7 @@ def test_read_only_casual_user_privileges(self): config.save() # get user - user = get_user_model().objects.get(username='user1') + user, _ = get_user_model().objects.get_or_create(username='user1') web_client.force_login(user) # post not whitelisted URL as superuser @@ -788,6 +788,7 @@ def test_display_change_perms_button_tag_standard(self): class TestGetVisibleResource(TestCase): + def setUp(self): self.user = get_user_model().objects.create(username='mikel_arteta') self.category = TopicCategory.objects.create(identifier='biota') @@ -857,6 +858,7 @@ def test_converted_html_in_tags_with_with_multiple_tags(self): class TestTagThesaurus(TestCase): + # loading test thesausurs fixtures = [ "test_thesaurus.json" @@ -927,6 +929,7 @@ def __get_last_thesaurus(): @override_settings(THESAURUS_DEFAULT_LANG="en") class TestThesaurusAvailableForm(TestCase): + # loading test thesausurs fixtures = [ "test_thesaurus.json" @@ -983,38 +986,42 @@ def test_will_return_thesaurus_with_the_defaul_order_as_0(self): class TestFacets(TestCase): + def setUp(self): self.user = get_user_model().objects.create(username='test', email='test@test.com') Layer.objects.create( - owner=self.user, title='test_boxes', abstract='nothing', storeType='dataStore', is_approved=True + owner=self.user, title='test_boxes', abstract='nothing', storetype='vector', is_approved=True ) Layer.objects.create( - owner=self.user, title='test_1', abstract='contains boxes', storeType='dataStore', is_approved=True + owner=self.user, title='test_1', abstract='contains boxes', storetype='vector', is_approved=True ) Layer.objects.create( - owner=self.user, title='test_2', purpose='contains boxes', storeType='dataStore', is_approved=True + owner=self.user, title='test_2', purpose='contains boxes', storetype='vector', is_approved=True ) Layer.objects.create( - owner=self.user, title='test_3', storeType='dataStore', is_approved=True + owner=self.user, title='test_3', storetype='vector', is_approved=True ) Layer.objects.create( - owner=self.user, title='test_boxes', abstract='nothing', storeType='coverageStore', is_approved=True + owner=self.user, title='test_boxes', abstract='nothing', storetype='raster', is_approved=True ) Layer.objects.create( - owner=self.user, title='test_1', abstract='contains boxes', storeType='coverageStore', is_approved=True + owner=self.user, title='test_1', abstract='contains boxes', storetype='raster', is_approved=True ) Layer.objects.create( - owner=self.user, title='test_2', purpose='contains boxes', storeType='coverageStore', is_approved=True + owner=self.user, title='test_2', purpose='contains boxes', storetype='raster', is_approved=True ) Layer.objects.create( - owner=self.user, title='test_boxes', storeType='coverageStore', is_approved=True + owner=self.user, title='test_boxes', storetype='raster', is_approved=True ) self.request_mock = Mock(spec=requests.Request, GET=Mock()) def test_facets_filter_layers_returns_correctly(self): - ResourceBase.objects.all().update(dirty_state=False) + for _l in Layer.objects.all(): + _l.set_default_permissions() + _l.clear_dirty_state() + _l.set_processing_state(enumerations.STATE_PROCESSED) self.request_mock.GET.get.side_effect = lambda key, self: { 'title__icontains': 'boxes', 'abstract__icontains': 'boxes', @@ -1032,6 +1039,7 @@ def test_facets_filter_layers_returns_correctly(self): class TestGenerateThesaurusReference(TestCase): + fixtures = [ "test_thesaurus.json" ] diff --git a/geonode/base/thumb_utils.py b/geonode/base/thumb_utils.py index 356dfdb3abf..f3b2897dc9e 100644 --- a/geonode/base/thumb_utils.py +++ b/geonode/base/thumb_utils.py @@ -1,7 +1,7 @@ import os from django.conf import settings -from django.core.files.storage import default_storage as storage +from geonode.storage.manager import storage_manager def thumb_path(filename): @@ -12,13 +12,13 @@ def thumb_path(filename): def thumb_exists(filename): """Determine if a thumbnail file exists in storage""" - return storage.exists(thumb_path(filename)) + return storage_manager.exists(thumb_path(filename)) def thumb_size(filepath): """Determine if a thumbnail file size in storage""" - if storage.exists(filepath): - return storage.size(filepath) + if storage_manager.exists(filepath): + return storage_manager.size(filepath) elif os.path.exists(filepath): return os.path.getsize(filepath) return 0 @@ -26,20 +26,20 @@ def thumb_size(filepath): def thumb_open(filename): """Returns file handler of a thumbnail on the storage""" - return storage.open(thumb_path(filename)) + return storage_manager.open(thumb_path(filename)) def get_thumbs(): """Fetches a list of all stored thumbnails""" - if not storage.exists(settings.THUMBNAIL_LOCATION): + if not storage_manager.exists(settings.THUMBNAIL_LOCATION): return [] - subdirs, thumbs = storage.listdir(settings.THUMBNAIL_LOCATION) + subdirs, thumbs = storage_manager.listdir(settings.THUMBNAIL_LOCATION) return thumbs def remove_thumb(filename): """Delete a thumbnail from storage""" - storage.delete(thumb_path(filename)) + storage_manager.delete(thumb_path(filename)) def remove_thumbs(name): diff --git a/geonode/base/translation.py b/geonode/base/translation.py index a6d0a1c5a03..9c914afe8c7 100755 --- a/geonode/base/translation.py +++ b/geonode/base/translation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/base/urls.py b/geonode/base/urls.py index 3a7ade5251d..c28e4b43522 100644 --- a/geonode/base/urls.py +++ b/geonode/base/urls.py @@ -1,4 +1,3 @@ - ######################################################################### # # Copyright (C) 2019 OSGeo @@ -20,8 +19,13 @@ from django.conf.urls import url, include from .views import ( - ResourceBaseAutocomplete, RegionAutocomplete, - HierarchicalKeywordAutocomplete, ThesaurusKeywordLabelAutocomplete, OwnerRightsRequestView, ThesaurusAvailable) + resource_clone, + RegionAutocomplete, + ThesaurusAvailable, + OwnerRightsRequestView, + ResourceBaseAutocomplete, + HierarchicalKeywordAutocomplete, + ThesaurusKeywordLabelAutocomplete) urlpatterns = [ @@ -58,5 +62,10 @@ OwnerRightsRequestView.as_view(), name='owner_rights_request', ), + url( + r'^resource_clone/?$', + resource_clone, + name='resource_clone', + ), url(r'^', include('geonode.base.api.urls')), ] diff --git a/geonode/base/utils.py b/geonode/base/utils.py index 375e5c206da..ff6d9e8cd92 100644 --- a/geonode/base/utils.py +++ b/geonode/base/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -36,11 +35,8 @@ # Geonode functionality from guardian.shortcuts import get_perms, remove_perm, assign_perm -from geonode.documents.models import Document from geonode.layers.models import Layer from geonode.base.models import ResourceBase, Link, Configuration -from geonode.maps.models import Map -from geonode.services.models import Service from geonode.base.thumb_utils import ( get_thumbs, remove_thumb) @@ -165,14 +161,7 @@ def get_message_recipients(owner): @staticmethod def get_resource(resource_base): - if resource_base.polymorphic_ctype.name == 'layer': - return Layer.objects.get(pk=resource_base.pk) - elif resource_base.polymorphic_ctype.name == 'document': - return Document.objects.get(pk=resource_base.pk) - elif resource_base.polymorphic_ctype.name == 'map': - return Map.objects.get(pk=resource_base.pk) - else: - return Service.objects.get(pk=resource_base.pk) + return resource_base.get_real_instance() @staticmethod def is_admin_publish_mode(): diff --git a/geonode/base/views.py b/geonode/base/views.py index 4231b7c768e..9b42691ff9f 100644 --- a/geonode/base/views.py +++ b/geonode/base/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -17,30 +16,35 @@ # along with this program. If not, see . # ######################################################################### +import json +import logging +from dal import views, autocomplete +from user_messages.models import Message +from guardian.shortcuts import get_objects_for_user -# Geonode functionality -from django.shortcuts import render from django.conf import settings +from django.http import Http404 +from django.shortcuts import render from django.http import HttpResponse from django.views.generic import FormView from django.http import HttpResponseRedirect from django.contrib.auth import get_user_model from django.utils.translation import ugettext as _ from django.core.exceptions import PermissionDenied +from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin -from dal import views, autocomplete -from user_messages.models import Message -from guardian.shortcuts import get_objects_for_user - +# Geonode dependencies from geonode.maps.models import Map from geonode.layers.models import Layer from geonode.utils import resolve_object +from geonode.monitoring import register_event from geonode.documents.models import Document from geonode.groups.models import GroupProfile from geonode.tasks.tasks import set_permissions from geonode.base.forms import CuratedThumbnailForm +from geonode.resource.manager import resource_manager from geonode.security.utils import get_visible_resources from geonode.notifications_helper import send_notification from geonode.base.utils import OwnerRightsRequestViewUtils @@ -53,10 +57,13 @@ from geonode.base.models import ( Region, ResourceBase, - HierarchicalKeyword, ThesaurusKeyword, + HierarchicalKeyword, + ThesaurusKeyword, ThesaurusKeywordLabel ) +logger = logging.getLogger(__name__) + def user_and_group_permission(request, model): if not request.user.is_superuser: @@ -397,3 +404,44 @@ def post(self, request, *args, **kwargs): return self.form_valid(form) else: return self.form_invalid(form) + + +@login_required +def resource_clone(request): + try: + uuid = request.POST['uuid'] + resource = resolve_object( + request, ResourceBase, { + 'uuid': uuid}, 'base.change_resourcebase') + except PermissionDenied: + return HttpResponse("Not allowed", status=403) + except Exception: + raise Http404("Not found") + if not resource: + raise Http404("Not found") + + out = {} + try: + getattr(resource_manager, "copy")( + resource.get_real_instance(), + uuid=None, + defaults={ + 'user': request.user}) + out['success'] = True + out['message'] = _("Resource Cloned Successfully!") + except Exception as e: + logger.exception(e) + out['success'] = False + out['message'] = _(f"Error Occurred while Cloning the Resource: {e}") + out['errors'] = str(e) + + if out['success']: + status_code = 200 + register_event(request, 'change', resource) + else: + status_code = 400 + + return HttpResponse( + json.dumps(out), + content_type='application/json', + status=status_code) diff --git a/geonode/base/widgets.py b/geonode/base/widgets.py index f0b57415ea0..08e41930e23 100644 --- a/geonode/base/widgets.py +++ b/geonode/base/widgets.py @@ -1,4 +1,3 @@ - from dal_select2_taggit.widgets import TaggitSelect2 @@ -15,7 +14,7 @@ def value_from_datadict(self, data, files, name): """ try: - value = super(TaggitSelect2Custom, self).value_from_datadict(data, files, name) + value = super().value_from_datadict(data, files, name) if value and ',' not in value: value = f'{value},' return value diff --git a/geonode/br/__init__.py b/geonode/br/__init__.py index 85cbc5c6a33..5094cf7ca78 100644 --- a/geonode/br/__init__.py +++ b/geonode/br/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/br/admin.py b/geonode/br/admin.py index 10a7fee78ef..97c222d35dc 100644 --- a/geonode/br/admin.py +++ b/geonode/br/admin.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/br/management/__init__.py b/geonode/br/management/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/br/management/__init__.py +++ b/geonode/br/management/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/br/management/commands/__init__.py b/geonode/br/management/commands/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/br/management/commands/__init__.py +++ b/geonode/br/management/commands/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/br/management/commands/backup.py b/geonode/br/management/commands/backup.py index bc0b1889c16..c7aba93204b 100644 --- a/geonode/br/management/commands/backup.py +++ b/geonode/br/management/commands/backup.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo diff --git a/geonode/br/management/commands/restore.py b/geonode/br/management/commands/restore.py index 94f07019b44..b88b57002c0 100755 --- a/geonode/br/management/commands/restore.py +++ b/geonode/br/management/commands/restore.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo @@ -268,20 +267,20 @@ def execute_restore(self, **options): locale_files_folders = os.path.join(target_folder, utils.LOCALE_PATHS) try: - print((f"[Sanity Check] Full Write Access to '{restore_folder}' ...")) + print(f"[Sanity Check] Full Write Access to '{restore_folder}' ...") chmod_tree(restore_folder) - print((f"[Sanity Check] Full Write Access to '{media_root}' ...")) + print(f"[Sanity Check] Full Write Access to '{media_root}' ...") chmod_tree(media_root) - print((f"[Sanity Check] Full Write Access to '{static_root}' ...")) + print(f"[Sanity Check] Full Write Access to '{static_root}' ...") chmod_tree(static_root) for static_files_folder in static_folders: - print((f"[Sanity Check] Full Write Access to '{static_files_folder}' ...")) + print(f"[Sanity Check] Full Write Access to '{static_files_folder}' ...") chmod_tree(static_files_folder) for template_files_folder in template_folders: - print((f"[Sanity Check] Full Write Access to '{template_files_folder}' ...")) + print(f"[Sanity Check] Full Write Access to '{template_files_folder}' ...") chmod_tree(template_files_folder) for locale_files_folder in locale_folders: - print((f"[Sanity Check] Full Write Access to '{locale_files_folder}' ...")) + print(f"[Sanity Check] Full Write Access to '{locale_files_folder}' ...") chmod_tree(locale_files_folder) except Exception as exception: if notify: @@ -294,7 +293,7 @@ def execute_restore(self, **options): if not skip_geoserver: try: - print((f"[Sanity Check] Full Write Access to '{target_folder}' ...")) + print(f"[Sanity Check] Full Write Access to '{target_folder}' ...") chmod_tree(target_folder) self.restore_geoserver_backup(config, settings, target_folder, skip_geoserver_info, skip_geoserver_security, @@ -599,7 +598,7 @@ def validate_backup_file_hash(self, backup_file: str) -> str: archive_md5_file = f"{backup_file.rsplit('.', 1)[0]}.md5" if os.path.exists(archive_md5_file): - with open(archive_md5_file, 'r') as md5_file: + with open(archive_md5_file) as md5_file: original_backup_md5 = md5_file.readline().strip().split(" ")[0] if original_backup_md5 != backup_hash: @@ -642,7 +641,7 @@ def restore_geoserver_backup(self, config, settings, target_folder, geoserver_bk_file = os.path.join(target_folder, 'geoserver_catalog.zip') if not os.path.exists(geoserver_bk_file) or not os.access(geoserver_bk_file, os.R_OK): - raise Exception((f'ERROR: geoserver restore: file "{geoserver_bk_file}" not found.')) + raise Exception(f'ERROR: geoserver restore: file "{geoserver_bk_file}" not found.') print(f"Restoring 'GeoServer Catalog [{url}]' from '{geoserver_bk_file}'.") diff --git a/geonode/br/management/commands/utils/utils.py b/geonode/br/management/commands/utils/utils.py index a664d136e92..47c1a21f6e0 100644 --- a/geonode/br/management/commands/utils/utils.py +++ b/geonode/br/management/commands/utils/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo @@ -89,7 +88,7 @@ def geoserver_option_list(parser): help="Don't dump geoserver raster data") -class Config(object): +class Config: def __init__(self, options): self.config_parser = None diff --git a/geonode/br/migrations/__init__.py b/geonode/br/migrations/__init__.py index 735fc7c7f22..71f8f62811e 100644 --- a/geonode/br/migrations/__init__.py +++ b/geonode/br/migrations/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/br/models.py b/geonode/br/models.py index 54f846ab60b..c1c8f0ec0bc 100644 --- a/geonode/br/models.py +++ b/geonode/br/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/br/tests/__init__.py b/geonode/br/tests/__init__.py index dd2ade49026..7fb28c2fa8c 100644 --- a/geonode/br/tests/__init__.py +++ b/geonode/br/tests/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/br/tests/factories.py b/geonode/br/tests/factories.py index 41127284cb0..87f13cc7af5 100644 --- a/geonode/br/tests/factories.py +++ b/geonode/br/tests/factories.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/br/tests/test_backup.py b/geonode/br/tests/test_backup.py index e6b8439b7b1..3867ebf4750 100644 --- a/geonode/br/tests/test_backup.py +++ b/geonode/br/tests/test_backup.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -19,7 +18,7 @@ ######################################################################### import os -import mock +from unittest import mock import tempfile from django.core.management import call_command diff --git a/geonode/br/tests/test_restore.py b/geonode/br/tests/test_restore.py index 0eac15d0953..d31fadc3dfa 100644 --- a/geonode/br/tests/test_restore.py +++ b/geonode/br/tests/test_restore.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -19,7 +18,7 @@ ######################################################################### import os -import mock +from unittest import mock import zipfile import tempfile diff --git a/geonode/br/tests/test_restore_helpers.py b/geonode/br/tests/test_restore_helpers.py index 6550a6b73d1..c5fbb09d061 100644 --- a/geonode/br/tests/test_restore_helpers.py +++ b/geonode/br/tests/test_restore_helpers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/__init__.py b/geonode/catalogue/__init__.py index 85a908011bf..1e015e43390 100644 --- a/geonode/catalogue/__init__.py +++ b/geonode/catalogue/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -49,7 +48,7 @@ def load_backend(backend_name): try: available_backends = sorted([f for f in os.listdir(backend_dir) if os.path.isdir(os.path.join(backend_dir, f)) and not f.startswith('.')]) - except EnvironmentError: + except OSError: available_backends = [] if backend_name not in available_backends: diff --git a/geonode/catalogue/backends/__init__.py b/geonode/catalogue/backends/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/catalogue/backends/__init__.py +++ b/geonode/catalogue/backends/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/backends/base.py b/geonode/catalogue/backends/base.py index 406d22cf33e..fafcdfcac74 100644 --- a/geonode/catalogue/backends/base.py +++ b/geonode/catalogue/backends/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/backends/generic.py b/geonode/catalogue/backends/generic.py index 69bdcbc2c2f..2b91a295c1a 100644 --- a/geonode/catalogue/backends/generic.py +++ b/geonode/catalogue/backends/generic.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/backends/pycsw_http.py b/geonode/catalogue/backends/pycsw_http.py index 7013f09a70c..ae7a29219d2 100644 --- a/geonode/catalogue/backends/pycsw_http.py +++ b/geonode/catalogue/backends/pycsw_http.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -27,6 +26,6 @@ class CatalogueBackend(GenericCatalogueBackend): def __init__(self, *args, **kwargs): """initialize pycsw HTTP CSW backend""" - super(CatalogueBackend, self).__init__(*args, **kwargs) + super(CatalogueBackend, self).__init__(*args, **kwargs) # LGTM: super() will not work in old-style classes self.catalogue.formats = \ ['Atom', 'DIF', 'Dublin Core', 'ebRIM', 'FGDC', 'ISO'] diff --git a/geonode/catalogue/backends/pycsw_local.py b/geonode/catalogue/backends/pycsw_local.py index 0f4fac360d6..363cff4ecf1 100644 --- a/geonode/catalogue/backends/pycsw_local.py +++ b/geonode/catalogue/backends/pycsw_local.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -58,8 +57,9 @@ class CatalogueBackend(GenericCatalogueBackend): + def __init__(self, *args, **kwargs): - super(CatalogueBackend, self).__init__(*args, **kwargs) + super(CatalogueBackend, self).__init__(*args, **kwargs) # LGTM: super() will not work in old-style classes self.catalogue.formats = ['Atom', 'DIF', 'Dublin Core', 'ebRIM', 'FGDC', 'ISO'] self.catalogue.local = True diff --git a/geonode/catalogue/backends/pycsw_local_mappings.py b/geonode/catalogue/backends/pycsw_local_mappings.py index 6301502da13..ead12f37b9a 100644 --- a/geonode/catalogue/backends/pycsw_local_mappings.py +++ b/geonode/catalogue/backends/pycsw_local_mappings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/backends/pycsw_plugin.py b/geonode/catalogue/backends/pycsw_plugin.py index 6c0daa9aac0..aa7ad027fec 100644 --- a/geonode/catalogue/backends/pycsw_plugin.py +++ b/geonode/catalogue/backends/pycsw_plugin.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo diff --git a/geonode/catalogue/metadataxsl/__init__.py b/geonode/catalogue/metadataxsl/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/catalogue/metadataxsl/__init__.py +++ b/geonode/catalogue/metadataxsl/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/metadataxsl/management/__init__.py b/geonode/catalogue/metadataxsl/management/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/catalogue/metadataxsl/management/__init__.py +++ b/geonode/catalogue/metadataxsl/management/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/metadataxsl/management/commands/__init__.py b/geonode/catalogue/metadataxsl/management/commands/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/catalogue/metadataxsl/management/commands/__init__.py +++ b/geonode/catalogue/metadataxsl/management/commands/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/metadataxsl/management/commands/addmissinglinks.py b/geonode/catalogue/metadataxsl/management/commands/addmissinglinks.py index 999b178b9bd..919cfdad3c9 100644 --- a/geonode/catalogue/metadataxsl/management/commands/addmissinglinks.py +++ b/geonode/catalogue/metadataxsl/management/commands/addmissinglinks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/metadataxsl/models.py b/geonode/catalogue/metadataxsl/models.py index 95fd19385d1..7d41fa0ab20 100644 --- a/geonode/catalogue/metadataxsl/models.py +++ b/geonode/catalogue/metadataxsl/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/metadataxsl/tests.py b/geonode/catalogue/metadataxsl/tests.py index 42bad457c39..7795a315eb7 100644 --- a/geonode/catalogue/metadataxsl/tests.py +++ b/geonode/catalogue/metadataxsl/tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -28,6 +27,6 @@ class MetadataXSLTest(GeoNodeBaseTestSupport): """ def setUp(self): - super(MetadataXSLTest, self).setUp() + super().setUp() self.adm_un = "admin" self.adm_pw = "admin" diff --git a/geonode/catalogue/metadataxsl/urls.py b/geonode/catalogue/metadataxsl/urls.py index 35f3e242686..8e6b4348df7 100644 --- a/geonode/catalogue/metadataxsl/urls.py +++ b/geonode/catalogue/metadataxsl/urls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/metadataxsl/views.py b/geonode/catalogue/metadataxsl/views.py index 042177842e9..c2fc30fd9c5 100644 --- a/geonode/catalogue/metadataxsl/views.py +++ b/geonode/catalogue/metadataxsl/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/models.py b/geonode/catalogue/models.py index 3d17ff94cf8..134a109b8e3 100644 --- a/geonode/catalogue/models.py +++ b/geonode/catalogue/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -48,7 +47,7 @@ def catalogue_post_save(instance, sender, **kwargs): catalogue = get_catalogue() catalogue.create_record(instance) record = catalogue.get_record(instance.uuid) - except EnvironmentError as err: + except OSError as err: msg = f'Could not connect to catalogue to save information for layer "{instance.name}"' if err.errno == errno.ECONNREFUSED: LOGGER.warn(msg, err) diff --git a/geonode/catalogue/templates/catalogue/full_metadata.xml b/geonode/catalogue/templates/catalogue/full_metadata.xml index 4fdebeb2634..0d4a88d653e 100644 --- a/geonode/catalogue/templates/catalogue/full_metadata.xml +++ b/geonode/catalogue/templates/catalogue/full_metadata.xml @@ -326,7 +326,7 @@ - {% if layer.store_type == 'coverageStore' %} + {% if layer.store_type == 'raster' %} GeoTIFF {% else %} ESRI Shapefile @@ -493,14 +493,14 @@ - {% if layer.storeType == 'coverageStore' %} + {% if layer.storetype == 'raster' %} image - {% elif layer.storeType == 'dataStore' %} + {% elif layer.storetype == 'vector' %} 0 diff --git a/geonode/catalogue/tests.py b/geonode/catalogue/tests.py index fd2c288afdf..03008cd0437 100644 --- a/geonode/catalogue/tests.py +++ b/geonode/catalogue/tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -17,30 +16,51 @@ # along with this program. If not, see . # ######################################################################### -from django.contrib.auth import get_user_model -from geonode.layers.populate_layers_data import create_layer_data -from geonode.catalogue.views import csw_global_dispatch import logging -from django.contrib.auth.models import AnonymousUser import xml.etree.ElementTree as ET -from geonode.layers.models import Layer -from geonode.tests.base import GeoNodeBaseTestSupport +from django.db.models import Q from django.test import RequestFactory +from geonode.layers.models import Layer from geonode.catalogue import get_catalogue +from django.contrib.auth import get_user_model +from django.contrib.auth.models import AnonymousUser +from geonode.tests.base import GeoNodeBaseTestSupport from geonode.catalogue.models import catalogue_post_save -from django.db.models import Q + +from geonode.catalogue.views import csw_global_dispatch +from geonode.layers.populate_layers_data import create_layer_data + +from geonode.base.populate_test_data import ( + all_public, + create_models, + remove_models) logger = logging.getLogger(__name__) class CatalogueTest(GeoNodeBaseTestSupport): + + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + def setUp(self): - super(CatalogueTest, self).setUp() + super().setUp() + self.request = self.__request_factory_single(123) + create_layer_data() + self.user = "admin" + self.passwd = "admin" def test_get_catalog(self): """Tests the get_catalogue function works.""" - c = get_catalogue() self.assertIsNotNone(c) @@ -62,15 +82,6 @@ def test_update_metadata_records(self): if len(record.identification.otherconstraints) > 0: self.assertEqual(record.identification.otherconstraints[0], layer.raw_constraints_other) - -class TestCswGlobalDispatch(GeoNodeBaseTestSupport): - def setUp(self): - super(TestCswGlobalDispatch, self).setUp() - self.request = self.__request_factory_single(123) - create_layer_data() - self.user = "admin" - self.passwd = "admin" - def test_given_a_simple_request_should_return_200(self): actual = csw_global_dispatch(self.request) self.assertEqual(200, actual.status_code) diff --git a/geonode/catalogue/urls.py b/geonode/catalogue/urls.py index 69a8fcd9452..7ea79e56a27 100644 --- a/geonode/catalogue/urls.py +++ b/geonode/catalogue/urls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/catalogue/views.py b/geonode/catalogue/views.py index 1c4f64df463..e45dc6eed33 100644 --- a/geonode/catalogue/views.py +++ b/geonode/catalogue/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/celery_app.py b/geonode/celery_app.py index a849bf10dac..a299fb00012 100644 --- a/geonode/celery_app.py +++ b/geonode/celery_app.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -36,6 +35,7 @@ def _log(msg, *args): # pickle the object when using Windows. app.config_from_object('django.conf:settings', namespace="CELERY") app.autodiscover_tasks() +app.autodiscover_tasks(packages=["geonode.harvesting.harvesters"]) """ CELERAY SAMPLE TASKS @app.on_after_configure.connect diff --git a/geonode/client/__init__.py b/geonode/client/__init__.py index a0f3a990243..55702cb7715 100644 --- a/geonode/client/__init__.py +++ b/geonode/client/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo diff --git a/geonode/client/apps.py b/geonode/client/apps.py index 6615a4099b4..a6f48b9eab7 100644 --- a/geonode/client/apps.py +++ b/geonode/client/apps.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo diff --git a/geonode/client/conf.py b/geonode/client/conf.py index 95bce262fa6..1da7749f2a3 100644 --- a/geonode/client/conf.py +++ b/geonode/client/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo diff --git a/geonode/client/hooks.py b/geonode/client/hooks.py index b3e68bf59d1..d651a416d3d 100644 --- a/geonode/client/hooks.py +++ b/geonode/client/hooks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo @@ -20,7 +19,7 @@ from .conf import settings -class HookProxy(object): +class HookProxy: def __getattr__(self, attr): if not isinstance(settings.GEONODE_CLIENT_HOOKSET, str): diff --git a/geonode/client/hooksets.py b/geonode/client/hooksets.py index 155f7326733..4d1d089d540 100644 --- a/geonode/client/hooksets.py +++ b/geonode/client/hooksets.py @@ -1,112 +1,148 @@ -# -*- coding: utf-8 -*- -######################################################################### -# -# Copyright (C) 2018 OSGeo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -######################################################################### -import json - - -class BaseHookSet(object): - - # Layers - def layer_list_template(self, context=None): - return 'layers/layer_list_default.html' - - def layer_detail_template(self, context=None): - return NotImplemented - - def layer_new_template(self, context=None): - return NotImplemented - - def layer_view_template(self, context=None): - return NotImplemented - - def layer_edit_template(self, context=None): - return NotImplemented - - def layer_update_template(self, context=None): - return NotImplemented - - def layer_embed_template(self, context=None): - return NotImplemented - - def layer_download_template(self, context=None): - return NotImplemented - - def layer_style_edit_template(self, context=None): - return NotImplemented - - # Maps - def map_list_template(self, context=None): - return 'maps/map_list_default.html' - - def map_detail_template(self, context=None): - return NotImplemented - - def map_new_template(self, context=None): - return NotImplemented - - def map_view_template(self, context=None): - return NotImplemented - - def map_edit_template(self, context=None): - return NotImplemented - - def map_update_template(self, context=None): - return NotImplemented - - def map_embed_template(self, context=None): - return NotImplemented - - def map_download_template(self, context=None): - return NotImplemented - - # GeoApps - def geoapp_list_template(self, context=None): - return 'apps/app_list_default.html' - - def geoapp_detail_template(self, context=None): - return NotImplemented - - def geoapp_new_template(self, context=None): - return NotImplemented - - def geoapp_view_template(self, context=None): - return NotImplemented - - def geoapp_edit_template(self, context=None): - return NotImplemented - - def geoapp_update_template(self, context=None): - return NotImplemented - - def geoapp_embed_template(self, context=None): - return NotImplemented - - def geoapp_download_template(self, context=None): - return NotImplemented - - # Map Persisting - def viewer_json(self, conf, context=None): - if isinstance(conf, str): - conf = json.loads(conf) - return conf - - def update_from_viewer(self, conf, context=None): - conf = self.viewer_json(conf, context=context) - context['config'] = conf - return 'maps/map_edit.html' +######################################################################### +# +# Copyright (C) 2018 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import json + +from django.conf import settings +from django.urls.base import reverse + + +class BaseHookSet: + + # Layers + def layer_list_template(self, context=None): + return 'layers/layer_list_default.html' + + def layer_detail_template(self, context=None): + return NotImplemented + + def layer_new_template(self, context=None): + return NotImplemented + + def layer_view_template(self, context=None): + return NotImplemented + + def layer_edit_template(self, context=None): + return NotImplemented + + def layer_update_template(self, context=None): + return NotImplemented + + def layer_embed_template(self, context=None): + return NotImplemented + + def layer_download_template(self, context=None): + return NotImplemented + + def layer_style_edit_template(self, context=None): + return NotImplemented + + def layer_list_url(self): + return self.add_limit_settings(reverse('layer_browse')) + + def layer_detail_url(self, layer): + return reverse('layer_detail', args=(layer.alternate,)) + + # Maps + def map_list_template(self, context=None): + return 'maps/map_list_default.html' + + def map_detail_template(self, context=None): + return NotImplemented + + def map_new_template(self, context=None): + return NotImplemented + + def map_view_template(self, context=None): + return NotImplemented + + def map_edit_template(self, context=None): + return NotImplemented + + def map_update_template(self, context=None): + return NotImplemented + + def map_embed_template(self, context=None): + return NotImplemented + + def map_download_template(self, context=None): + return NotImplemented + + def map_list_url(self): + return self.add_limit_settings(reverse('maps_browse')) + + def map_detail_url(self, map): + return reverse('map_detail', args=(map.id,)) + + # GeoApps + def geoapp_list_template(self, context=None): + return 'apps/app_list_default.html' + + def geoapp_detail_template(self, context=None): + return NotImplemented + + def geoapp_new_template(self, context=None): + return NotImplemented + + def geoapp_view_template(self, context=None): + return NotImplemented + + def geoapp_edit_template(self, context=None): + return NotImplemented + + def geoapp_update_template(self, context=None): + return NotImplemented + + def geoapp_embed_template(self, context=None): + return NotImplemented + + def geoapp_download_template(self, context=None): + return NotImplemented + + def geoapp_list_url(self): + return self.add_limit_settings(reverse('apps_browse')) + + def geoapp_detail_url(self, geoapp): + return reverse('geoapp_detail', args=(geoapp.id,)) + + # Documents + def document_list_url(self): + return self.add_limit_settings(reverse('document_browse')) + + def document_detail_url(self, document): + return reverse('document_detail', args=(document.id,)) + + # Map Persisting + def viewer_json(self, conf, context=None): + if isinstance(conf, str): + conf = json.loads(conf) + return conf + + def update_from_viewer(self, conf, context=None): + conf = self.viewer_json(conf, context=context) + context['config'] = conf + return 'maps/map_edit.html' + + def add_limit_settings(self, url): + CLIENT_RESULTS_LIMIT = settings.CLIENT_RESULTS_LIMIT + return f"{url}?limit={CLIENT_RESULTS_LIMIT}" + + def metadata_update_redirect(self, url): + if "metadata_uri" in url: + return url.replace('/metadata_uri', '') + return url.replace('/metadata', '') diff --git a/geonode/client/migrations/0001_initial.py b/geonode/client/migrations/0001_initial.py index c3979cee4e5..a04cf6dc2bf 100644 --- a/geonode/client/migrations/0001_initial.py +++ b/geonode/client/migrations/0001_initial.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/client/migrations/0002_auto_20180412_1039.py b/geonode/client/migrations/0002_auto_20180412_1039.py index c31e67963d8..64a6ca5e4e9 100644 --- a/geonode/client/migrations/0002_auto_20180412_1039.py +++ b/geonode/client/migrations/0002_auto_20180412_1039.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/client/migrations/0002_auto_20180412_1039_squashed_0005_auto_20181015_1201.py b/geonode/client/migrations/0002_auto_20180412_1039_squashed_0005_auto_20181015_1201.py index 45c240a0e8b..c842e75deff 100644 --- a/geonode/client/migrations/0002_auto_20180412_1039_squashed_0005_auto_20181015_1201.py +++ b/geonode/client/migrations/0002_auto_20180412_1039_squashed_0005_auto_20181015_1201.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-04-04 08:32 diff --git a/geonode/client/migrations/0003_geonodethemecustomization_jumbotron_welcome_hide.py b/geonode/client/migrations/0003_geonodethemecustomization_jumbotron_welcome_hide.py index 8a92ee1f421..3ad9548e8e1 100644 --- a/geonode/client/migrations/0003_geonodethemecustomization_jumbotron_welcome_hide.py +++ b/geonode/client/migrations/0003_geonodethemecustomization_jumbotron_welcome_hide.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.11 on 2018-04-16 09:49 diff --git a/geonode/client/migrations/0004_auto_20180416_1319.py b/geonode/client/migrations/0004_auto_20180416_1319.py index 8aaec1be296..30dd8213992 100644 --- a/geonode/client/migrations/0004_auto_20180416_1319.py +++ b/geonode/client/migrations/0004_auto_20180416_1319.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.11 on 2018-04-16 13:19 diff --git a/geonode/client/migrations/0005_auto_20181015_1201.py b/geonode/client/migrations/0005_auto_20181015_1201.py index 9716902643e..54bccb2dde9 100644 --- a/geonode/client/migrations/0005_auto_20181015_1201.py +++ b/geonode/client/migrations/0005_auto_20181015_1201.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.15 on 2018-10-15 00:01 diff --git a/geonode/client/models.py b/geonode/client/models.py index 3de6751bfd4..5c3ac917f78 100644 --- a/geonode/client/models.py +++ b/geonode/client/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo diff --git a/geonode/client/templatetags/__init__.py b/geonode/client/templatetags/__init__.py index 47a464af34e..68c7b7d1937 100644 --- a/geonode/client/templatetags/__init__.py +++ b/geonode/client/templatetags/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo diff --git a/geonode/client/templatetags/client_lib_tags.py b/geonode/client/templatetags/client_lib_tags.py index 3e3f431b9d3..558a9725e20 100644 --- a/geonode/client/templatetags/client_lib_tags.py +++ b/geonode/client/templatetags/client_lib_tags.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo @@ -40,6 +39,47 @@ def google_api_key(): return getattr(settings, "GOOGLE_API_KEY", None) +# For client single page links +@register.simple_tag +def layer_list_url(): + return hookset.layer_list_url() + + +@register.simple_tag +def layer_detail_url(layer): + return hookset.layer_detail_url(layer) + + +@register.simple_tag +def map_list_url(): + return hookset.map_list_url() + + +@register.simple_tag +def map_detail_url(map): + return hookset.map_detail_url(map) + + +@register.simple_tag +def document_list_url(): + return hookset.document_list_url() + + +@register.simple_tag +def document_detail_url(document): + return hookset.document_detail_url(document) + + +@register.simple_tag +def geoapp_list_url(): + return hookset.geoapp_list_url() + + +@register.simple_tag +def geoapp_detail_url(geoapp): + return hookset.geoapp_detail_url(geoapp) + + def parse_tag(token, parser): """ Generic template tag parser. diff --git a/geonode/client/tests.py b/geonode/client/tests.py index 163d1b7521a..8e156a6796e 100644 --- a/geonode/client/tests.py +++ b/geonode/client/tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2018 OSGeo diff --git a/geonode/compat.py b/geonode/compat.py index fef12aad892..82edb00ff4e 100644 --- a/geonode/compat.py +++ b/geonode/compat.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/context_processors.py b/geonode/context_processors.py index 77bab1c041d..14da43c7019 100644 --- a/geonode/context_processors.py +++ b/geonode/context_processors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/custom_translations.py b/geonode/custom_translations.py index 500eef84a90..9ab50b0ab89 100644 --- a/geonode/custom_translations.py +++ b/geonode/custom_translations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/decorators.py b/geonode/decorators.py index 8b59aa94e2f..254e9cd6b7f 100644 --- a/geonode/decorators.py +++ b/geonode/decorators.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/documents/__init__.py b/geonode/documents/__init__.py index 289473a6753..ed2b956f7f5 100644 --- a/geonode/documents/__init__.py +++ b/geonode/documents/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/documents/admin.py b/geonode/documents/admin.py index 8817d214aed..dca0103fbed 100644 --- a/geonode/documents/admin.py +++ b/geonode/documents/admin.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/documents/api/__init__.py b/geonode/documents/api/__init__.py index fe4e643c905..0044807c781 100644 --- a/geonode/documents/api/__init__.py +++ b/geonode/documents/api/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/documents/api/permissions.py b/geonode/documents/api/permissions.py index 70f5efcfc72..4de90590b69 100644 --- a/geonode/documents/api/permissions.py +++ b/geonode/documents/api/permissions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/documents/api/serializers.py b/geonode/documents/api/serializers.py index c8c358d4446..3c63f553054 100644 --- a/geonode/documents/api/serializers.py +++ b/geonode/documents/api/serializers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo @@ -29,7 +28,7 @@ class DocumentSerializer(ResourceBaseSerializer): def __init__(self, *args, **kwargs): # Instantiate the superclass normally - super(DocumentSerializer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class Meta: model = Document @@ -37,5 +36,5 @@ class Meta: view_name = 'documents-list' fields = ( 'pk', 'uuid', 'name', 'href', - 'doc_type', 'extension', 'mime_type' + 'storetype', 'extension', 'mime_type' ) diff --git a/geonode/documents/api/tests.py b/geonode/documents/api/tests.py index 978fac1102c..7069146fcb9 100644 --- a/geonode/documents/api/tests.py +++ b/geonode/documents/api/tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -17,33 +16,20 @@ # along with this program. If not, see . # ######################################################################### -import django import logging from urllib.parse import urljoin from django.urls import reverse -from django.conf.urls import url, include -from django.views.generic import TemplateView -from django.views.i18n import JavaScriptCatalog -from rest_framework.test import APITestCase, URLPatternsTestCase +from rest_framework.test import APITestCase -from geonode.api.urls import router -from geonode.services.views import services from geonode.documents.models import Document -from geonode.maps.views import map_embed -from geonode.geoapps.views import geoapp_edit -from geonode.layers.views import layer_upload, layer_embed -from geonode.documents.views import document_download, document_link - -from geonode import geoserver -from geonode.utils import check_ogc_backend from geonode.base.populate_test_data import create_models logger = logging.getLogger(__name__) -class DocumentsApiTests(APITestCase, URLPatternsTestCase): +class DocumentsApiTests(APITestCase): fixtures = [ 'initial_data.json', @@ -51,60 +37,6 @@ class DocumentsApiTests(APITestCase, URLPatternsTestCase): 'default_oauth_apps.json' ] - urlpatterns = [ - url(r'^home/$', - TemplateView.as_view(template_name='index.html'), - name='home'), - url(r'^help/$', - TemplateView.as_view(template_name='help.html'), - name='help'), - url(r"^account/", include("allauth.urls")), - url(r'^people/', include('geonode.people.urls')), - url(r'^api/v2/', include(router.urls)), - url(r'^api/v2/', include('geonode.api.urls')), - url(r'^api/v2/api-auth/', include('rest_framework.urls', namespace='geonode_rest_framework')), - url(r'^(?P\d+)/download/?$', document_download, name='document_download'), - url(r'^upload$', layer_upload, name='layer_upload'), - url(r'^$', - TemplateView.as_view(template_name='layers/layer_list.html'), - {'facet_type': 'layers', 'is_layer': True}, - name='layer_browse'), - url(r'^$', - TemplateView.as_view(template_name='maps/map_list.html'), - {'facet_type': 'maps', 'is_map': True}, - name='maps_browse'), - url(r'^$', - TemplateView.as_view(template_name='documents/document_list.html'), - {'facet_type': 'documents', 'is_document': True}, - name='document_browse'), - url(r'^$', - TemplateView.as_view(template_name='groups/group_list.html'), - name='group_list'), - url(r'^search/$', - TemplateView.as_view(template_name='search/search.html'), - name='search'), - url(r'^$', services, name='services'), - url(r'^invitations/', include( - 'geonode.invitations.urls', namespace='geonode.invitations')), - url(r'^i18n/', include(django.conf.urls.i18n), name="i18n"), - url(r'^jsi18n/$', JavaScriptCatalog.as_view(), {}, name='javascript-catalog'), - url(r'^(?P[^/]+)/embed$', map_embed, name='map_embed'), - url(r'^(?P[^/]+)/embed$', layer_embed, name='layer_embed'), - url(r'^(?P[^/]+)/embed$', geoapp_edit, {'template': 'apps/app_embed.html'}, name='geoapp_embed'), - url(r'^developer/$', TemplateView.as_view(template_name='developer.html'), name='developer'), - url(r'^about/$', TemplateView.as_view(template_name='about.html'), name='about'), - url(r'^(?P\d+)/link/?$', document_link, name='document_link'), - ] - - if check_ogc_backend(geoserver.BACKEND_PACKAGE): - from geonode.geoserver.views import layer_acls, resolve_user - urlpatterns += [ - url(r'^acls/?$', layer_acls, name='layer_acls'), - url(r'^acls_dep/?$', layer_acls, name='layer_acls_dep'), - url(r'^resolve_user/?$', resolve_user, name='layer_resolve_user'), - url(r'^resolve_user_dep/?$', resolve_user, name='layer_resolve_user_dep'), - ] - def setUp(self): create_models(b'document') create_models(b'map') @@ -120,10 +52,6 @@ def test_documents(self): self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 5) self.assertEqual(response.data['total'], 9) - - # Test embed_url is provided - self.assertIn('link', response.data['documents'][0]['embed_url']) - # Pagination self.assertEqual(len(response.data['documents']), 9) logger.debug(response.data) diff --git a/geonode/documents/api/urls.py b/geonode/documents/api/urls.py index e7cfede1898..882f8dd855d 100644 --- a/geonode/documents/api/urls.py +++ b/geonode/documents/api/urls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index 9d3a626dc57..0fcccd7da4e 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo @@ -53,7 +52,7 @@ class DocumentViewSet(DynamicModelViewSet): DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, ExtentFilter, DocumentPermissionsFilter ] - queryset = Document.objects.all() + queryset = Document.objects.all().order_by('-date') serializer_class = DocumentSerializer pagination_class = GeoNodeApiPagination diff --git a/geonode/documents/enumerations.py b/geonode/documents/enumerations.py index d26d0235106..8165a38fdc2 100644 --- a/geonode/documents/enumerations.py +++ b/geonode/documents/enumerations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/documents/exif/__init__.py b/geonode/documents/exif/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/documents/exif/__init__.py +++ b/geonode/documents/exif/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/documents/exif/tests.py b/geonode/documents/exif/tests.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/documents/exif/tests.py +++ b/geonode/documents/exif/tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/documents/exif/utils.py b/geonode/documents/exif/utils.py index 20d78bf850d..c488513357c 100644 --- a/geonode/documents/exif/utils.py +++ b/geonode/documents/exif/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -131,10 +130,12 @@ def exif_extract_metadata_doc(doc): if not doc: return None - if not doc.doc_file: + if not doc.files: return None - if os.path.splitext(doc.doc_file.name)[1].lower()[1:] in {"jpg", "jpeg"}: + _, ext = os.path.splitext(os.path.basename(doc.files[0])) + + if ext[1:] in {"jpg", "jpeg"}: from PIL import Image, ExifTags img = Image.open(doc.doc_file.path) exif_data = { diff --git a/geonode/documents/forms.py b/geonode/documents/forms.py index 62aedf9f65b..91312043b2f 100644 --- a/geonode/documents/forms.py +++ b/geonode/documents/forms.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -24,25 +23,25 @@ import json import logging +from modeltranslation.forms import TranslationModelForm + from django import forms -from django.utils.translation import ugettext as _ -from django.contrib.contenttypes.models import ContentType from django.conf import settings from django.forms import HiddenInput -from modeltranslation.forms import TranslationModelForm +from django.utils.translation import ugettext as _ +from django.contrib.contenttypes.models import ContentType -from geonode.documents.models import ( - Document, - DocumentResourceLink, - get_related_resources, -) from geonode.maps.models import Map from geonode.layers.models import Layer +from geonode.resource.utils import get_related_resources +from geonode.documents.models import ( + Document, + DocumentResourceLink) logger = logging.getLogger(__name__) -class DocumentFormMixin(object): +class DocumentFormMixin: def generate_link_choices(self, resources=None): @@ -86,12 +85,14 @@ def save_many2many(self, links_field='links'): class DocumentForm(ResourceBaseForm, DocumentFormMixin): + title = forms.CharField(required=False) + links = forms.MultipleChoiceField( label=_("Link to"), required=False) def __init__(self, *args, **kwargs): - super(DocumentForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.fields['links'].choices = self.generate_link_choices() self.fields['links'].initial = self.generate_link_values( resources=get_related_resources(self.instance) @@ -118,7 +119,7 @@ class Meta(ResourceBaseForm.Meta): 'object_id', 'doc_file', 'extension', - 'doc_type', + 'storetype', 'doc_url') @@ -130,19 +131,28 @@ class DocumentDescriptionForm(forms.Form): class DocumentReplaceForm(forms.ModelForm): + doc_file = forms.FileField( + label=_("File"), + required=False) + + files = forms.CharField( + label=_("File"), + required=False) + """ The form used to replace a document. """ class Meta: model = Document - fields = ['doc_file', 'doc_url'] + fields = ['doc_url'] + exclude = ['files'] def clean(self): """ Ensures the doc_file or the doc_url field is populated. """ - cleaned_data = super(DocumentReplaceForm, self).clean() + cleaned_data = super().clean() doc_file = self.cleaned_data.get('doc_file') doc_url = self.cleaned_data.get('doc_url') @@ -155,7 +165,7 @@ def clean(self): return cleaned_data - def clean_doc_file(self): + def clean_files(self): """ Ensures the doc_file is valid. """ @@ -185,6 +195,10 @@ class DocumentCreateForm(TranslationModelForm, DocumentFormMixin): label=_("Link to"), required=False) + doc_file = forms.FileField( + label=_("File"), + required=False) + class Meta: model = Document fields = ['title', 'doc_file', 'doc_url'] @@ -193,7 +207,7 @@ class Meta: } def __init__(self, *args, **kwargs): - super(DocumentCreateForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.fields['links'].choices = self.generate_link_choices() def clean_permissions(self): @@ -211,16 +225,16 @@ def clean(self): """ Ensures the doc_file or the doc_url field is populated. """ - cleaned_data = super(DocumentCreateForm, self).clean() + cleaned_data = super().clean() doc_file = self.cleaned_data.get('doc_file') doc_url = self.cleaned_data.get('doc_url') if not doc_file and not doc_url: - logger.debug("Document must be a file or url.") + logger.error("Document must be a file or url.") raise forms.ValidationError(_("Document must be a file or url.")) if doc_file and doc_url: - logger.debug("A document cannot have both a file and a url.") + logger.error("A document cannot have both a file and a url.") raise forms.ValidationError( _("A document cannot have both a file and a url.")) diff --git a/geonode/documents/management/__init__.py b/geonode/documents/management/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/documents/management/__init__.py +++ b/geonode/documents/management/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/documents/management/commands/delete_orphaned_files.py b/geonode/documents/management/commands/delete_orphaned_files.py index 95afb12f688..7398961ed84 100644 --- a/geonode/documents/management/commands/delete_orphaned_files.py +++ b/geonode/documents/management/commands/delete_orphaned_files.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/documents/migrations/0028_auto_20170801_1228.py b/geonode/documents/migrations/0028_auto_20170801_1228.py index ed6e8b85abc..c77b4b3a1ba 100644 --- a/geonode/documents/migrations/0028_auto_20170801_1228.py +++ b/geonode/documents/migrations/0028_auto_20170801_1228.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/documents/migrations/0028_auto_20170801_1228_squashed_0035_auto_20190404_0820.py b/geonode/documents/migrations/0028_auto_20170801_1228_squashed_0035_auto_20190404_0820.py index 9b54e1f8e49..79eb7621ce9 100644 --- a/geonode/documents/migrations/0028_auto_20170801_1228_squashed_0035_auto_20190404_0820.py +++ b/geonode/documents/migrations/0028_auto_20170801_1228_squashed_0035_auto_20190404_0820.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-04-04 08:24 diff --git a/geonode/documents/migrations/0029_auto_20180301_1947.py b/geonode/documents/migrations/0029_auto_20180301_1947.py index c90ef67d428..9c254063188 100644 --- a/geonode/documents/migrations/0029_auto_20180301_1947.py +++ b/geonode/documents/migrations/0029_auto_20180301_1947.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-03-02 01:47 diff --git a/geonode/documents/migrations/0029_auto_20190429_0831.py b/geonode/documents/migrations/0029_auto_20190429_0831.py index 32133cc9f47..3cea3f2bda7 100644 --- a/geonode/documents/migrations/0029_auto_20190429_0831.py +++ b/geonode/documents/migrations/0029_auto_20190429_0831.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-04-29 08:31 diff --git a/geonode/documents/migrations/0030_auto_20180302_0430.py b/geonode/documents/migrations/0030_auto_20180302_0430.py index 3b0b0558cbd..7c45dec938e 100644 --- a/geonode/documents/migrations/0030_auto_20180302_0430.py +++ b/geonode/documents/migrations/0030_auto_20180302_0430.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2018-03-02 10:30 diff --git a/geonode/documents/migrations/0031_auto_20180409_1238.py b/geonode/documents/migrations/0031_auto_20180409_1238.py index 89db8b892c4..439489c4c61 100644 --- a/geonode/documents/migrations/0031_auto_20180409_1238.py +++ b/geonode/documents/migrations/0031_auto_20180409_1238.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.11 on 2018-04-09 12:38 diff --git a/geonode/documents/migrations/0032_auto_20180412_0822.py b/geonode/documents/migrations/0032_auto_20180412_0822.py index e94027d3ca1..1b09621c193 100644 --- a/geonode/documents/migrations/0032_auto_20180412_0822.py +++ b/geonode/documents/migrations/0032_auto_20180412_0822.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/documents/migrations/0032_remove_document_doc_file.py b/geonode/documents/migrations/0032_remove_document_doc_file.py new file mode 100644 index 00000000000..50485f4867d --- /dev/null +++ b/geonode/documents/migrations/0032_remove_document_doc_file.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2 on 2021-05-31 14:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0031_auto_20201107_2241'), + ] + + operations = [ + migrations.RemoveField( + model_name='document', + name='doc_file', + ), + ] diff --git a/geonode/documents/migrations/0033_auto_20180414_2120.py b/geonode/documents/migrations/0033_auto_20180414_2120.py index dc528c71d23..1d3255e1f14 100644 --- a/geonode/documents/migrations/0033_auto_20180414_2120.py +++ b/geonode/documents/migrations/0033_auto_20180414_2120.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.11 on 2018-04-14 21:20\ diff --git a/geonode/documents/migrations/0033_remove_document_doc_type.py b/geonode/documents/migrations/0033_remove_document_doc_type.py new file mode 100644 index 00000000000..2609d3c469a --- /dev/null +++ b/geonode/documents/migrations/0033_remove_document_doc_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-07-06 10:47 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0032_remove_document_doc_file'), + ('base', '0067_resourcebase_storetype'), + ] + + operations = [ + migrations.RemoveField( + model_name='document', + name='doc_type', + ), + ] diff --git a/geonode/documents/migrations/0034_auto_20190329_1652.py b/geonode/documents/migrations/0034_auto_20190329_1652.py index 2a8be4f9773..c5140ea8861 100644 --- a/geonode/documents/migrations/0034_auto_20190329_1652.py +++ b/geonode/documents/migrations/0034_auto_20190329_1652.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-03-29 16:52 diff --git a/geonode/documents/migrations/0035_auto_20190404_0820.py b/geonode/documents/migrations/0035_auto_20190404_0820.py index 0f69dd000a7..30995f741a9 100644 --- a/geonode/documents/migrations/0035_auto_20190404_0820.py +++ b/geonode/documents/migrations/0035_auto_20190404_0820.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-04-04 08:20 diff --git a/geonode/documents/migrations/24_initial.py b/geonode/documents/migrations/24_initial.py index cb7b1230cdd..3993aaa4a05 100644 --- a/geonode/documents/migrations/24_initial.py +++ b/geonode/documents/migrations/24_initial.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/documents/migrations/25_add_documentresourcelink_table.py b/geonode/documents/migrations/25_add_documentresourcelink_table.py index b4d4048e59c..bc79f3aad04 100644 --- a/geonode/documents/migrations/25_add_documentresourcelink_table.py +++ b/geonode/documents/migrations/25_add_documentresourcelink_table.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/documents/migrations/26_move_data_to_documentresourcelink_table.py b/geonode/documents/migrations/26_move_data_to_documentresourcelink_table.py index 19c80b978dc..efe50d85f28 100644 --- a/geonode/documents/migrations/26_move_data_to_documentresourcelink_table.py +++ b/geonode/documents/migrations/26_move_data_to_documentresourcelink_table.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from django.db import migrations diff --git a/geonode/documents/migrations/27_drop_resource_columns_from_document_table.py b/geonode/documents/migrations/27_drop_resource_columns_from_document_table.py index af036db060f..f59f16b1c2c 100644 --- a/geonode/documents/migrations/27_drop_resource_columns_from_document_table.py +++ b/geonode/documents/migrations/27_drop_resource_columns_from_document_table.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - - from django.db import migrations, models diff --git a/geonode/documents/models.py b/geonode/documents/models.py index 9cf0ad2a242..8d1e4b2cb7e 100644 --- a/geonode/documents/models.py +++ b/geonode/documents/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -17,30 +16,23 @@ # along with this program. If not, see . # ######################################################################### - -import os -import uuid import logging -from urllib.parse import urlparse, urljoin + +from urllib.parse import urljoin from django.conf import settings from django.db import models from django.urls import reverse -from django.db.models import signals from django.contrib.staticfiles import finders -from django.contrib.gis.geos import MultiPolygon from django.utils.translation import ugettext_lazy as _ from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.fields import GenericForeignKey -from uuid_upload_path import upload_to - +from geonode.maps.models import Map from geonode.layers.models import Layer -from geonode.base.models import ResourceBase, resourcebase_post_save, Link -from geonode.documents.enumerations import DOCUMENT_TYPE_MAP, DOCUMENT_MIMETYPE_MAP +from geonode.base.models import ResourceBase from geonode.maps.signals import map_changed_signal -from geonode.maps.models import Map -from geonode.security.utils import remove_object_permissions +from geonode.documents.enumerations import DOCUMENT_TYPE_MAP, DOCUMENT_MIMETYPE_MAP logger = logging.getLogger(__name__) @@ -51,17 +43,8 @@ class Document(ResourceBase): A document is any kind of information that can be attached to a map such as pdf, images, videos, xls... """ - doc_file = models.FileField( - upload_to=upload_to, - null=True, - blank=True, - max_length=255, - verbose_name=_('File')) - extension = models.CharField(max_length=128, blank=True, null=True) - doc_type = models.CharField(max_length=128, blank=True, null=True) - doc_url = models.URLField( blank=True, null=True, @@ -105,7 +88,7 @@ def find_placeholder(self): def href(self): if self.doc_url: return self.doc_url - elif self.doc_file: + elif self.files: return urljoin( settings.SITEURL, reverse('document_download', args=(self.id,)) @@ -113,7 +96,7 @@ def href(self): @property def is_file(self): - return self.doc_file and self.extension + return self.files and self.extension @property def mime_type(self): @@ -177,87 +160,6 @@ def get_related_documents(resource): return None -def get_related_resources(document): - if document.links: - try: - return [ - link.content_type.get_object_for_this_type(id=link.object_id) - for link in document.links.all() - ] - except Exception: - return [] - else: - return [] - - -def pre_save_document(instance, sender, **kwargs): - if instance.doc_file: - base_name, extension = os.path.splitext(instance.doc_file.name) - instance.extension = extension[1:] - doc_type_map = DOCUMENT_TYPE_MAP - doc_type_map.update(getattr(settings, 'DOCUMENT_TYPE_MAP', {})) - if doc_type_map is None: - doc_type = 'other' - else: - doc_type = doc_type_map.get( - instance.extension.lower(), 'other') - instance.doc_type = doc_type - - elif instance.doc_url: - if '.' in urlparse(instance.doc_url).path: - instance.extension = urlparse(instance.doc_url).path.rsplit('.')[-1] - - if not instance.uuid: - instance.uuid = str(uuid.uuid1()) - instance.csw_type = 'document' - - if instance.abstract == '' or instance.abstract is None: - instance.abstract = 'No abstract provided' - - if instance.title == '' or instance.title is None: - instance.title = instance.doc_file.name - - resources = get_related_resources(instance) - - # if there are (new) linked resources update the bbox computed by their bboxes - if resources: - bbox = MultiPolygon([r.bbox_polygon for r in resources]) - instance.set_bbox_polygon(bbox.extent, instance.srid) - elif not instance.bbox_polygon: - instance.set_bbox_polygon((-180, -90, 180, 90), '4326') - - -def post_save_document(instance, *args, **kwargs): - from .tasks import create_document_thumbnail - - name = None - ext = instance.extension - mime_type_map = DOCUMENT_MIMETYPE_MAP - mime_type_map.update(getattr(settings, 'DOCUMENT_MIMETYPE_MAP', {})) - mime = mime_type_map.get(ext, 'text/plain') - url = None - - if instance.id and instance.doc_file: - name = "Hosted Document" - site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith('http') else settings.SITEURL - url = f"{site_url}{reverse('document_download', args=(instance.id,))}" - create_document_thumbnail.apply_async((instance.id,)) - elif instance.doc_url: - name = "External Document" - url = instance.doc_url - - if name and url and ext: - Link.objects.get_or_create( - resource=instance.resourcebase_ptr, - url=url, - defaults=dict( - extension=ext, - name=name, - mime=mime, - url=url, - link_type='data',)) - - def update_documents_extent(sender, **kwargs): documents = get_related_documents(sender) if documents: @@ -265,12 +167,4 @@ def update_documents_extent(sender, **kwargs): document.save() -def pre_delete_document(instance, sender, **kwargs): - remove_object_permissions(instance.get_self_resource()) - - -signals.pre_save.connect(pre_save_document, sender=Document) -signals.post_save.connect(post_save_document, sender=Document) -signals.post_save.connect(resourcebase_post_save, sender=Document) -signals.pre_delete.connect(pre_delete_document, sender=Document) map_changed_signal.connect(update_documents_extent) diff --git a/geonode/documents/renderers.py b/geonode/documents/renderers.py index 5f75fb9ed67..dda0fc3e736 100644 --- a/geonode/documents/renderers.py +++ b/geonode/documents/renderers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2017 OSGeo diff --git a/geonode/documents/search_indexes.py b/geonode/documents/search_indexes.py index 6032fb6e508..7d5fa1ce391 100644 --- a/geonode/documents/search_indexes.py +++ b/geonode/documents/search_indexes.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/documents/tasks.py b/geonode/documents/tasks.py index 49afcec1026..77d4328a87f 100644 --- a/geonode/documents/tasks.py +++ b/geonode/documents/tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2017 OSGeo @@ -17,18 +16,18 @@ # along with this program. If not, see . # ######################################################################### - import os -from django.core.files.storage import default_storage as storage +from celery.utils.log import get_task_logger from geonode.celery_app import app -from celery.utils.log import get_task_logger +from geonode.storage.manager import storage_manager -from geonode.documents.models import Document -from geonode.documents.renderers import render_document -from geonode.documents.renderers import generate_thumbnail_content -from geonode.documents.renderers import ConversionError +from .models import Document +from .renderers import ( + render_document, + generate_thumbnail_content, + ConversionError) logger = get_task_logger(__name__) @@ -40,7 +39,7 @@ expires=600, acks_late=False, autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 5, 'countdown': 10}, + retry_kwargs={'max_retries': 2, 'countdown': 10}, retry_backoff=True, retry_backoff_max=700, retry_jitter=True) @@ -60,21 +59,19 @@ def create_document_thumbnail(self, object_id): image_file = None if document.is_image: - if not os.path.exists(storage.path(document.doc_file.name)): - from shutil import copyfile - copyfile( - document.doc_file.path, - storage.path(document.doc_file.name) - ) - image_file = storage.open(document.doc_file.name, 'rb') + dname = storage_manager.path(document.files[0]) + if storage_manager.exists(dname): + image_file = storage_manager.open(dname, 'rb') elif document.is_video or document.is_audio: image_file = open(document.find_placeholder(), 'rb') elif document.is_file: + dname = storage_manager.path(document.files[0]) try: - document_location = storage.path(document.doc_file.name) + document_location = storage_manager.path(dname) except NotImplementedError as e: logger.debug(e) - document_location = storage.url(document.doc_file.name) + + document_location = storage_manager.url(dname) try: image_path = render_document(document_location) @@ -95,7 +92,7 @@ def create_document_thumbnail(self, object_id): try: thumbnail_content = generate_thumbnail_content(image_file) except Exception as e: - logger.error(f"Could not generate thumbnail, falling back to 'placeholder': {e}") + logger.debug(f"Could not generate thumbnail, falling back to 'placeholder': {e}") thumbnail_content = generate_thumbnail_content(document.find_placeholder()) except Exception as e: logger.error(f"Could not generate thumbnail: {e}") @@ -121,7 +118,7 @@ def create_document_thumbnail(self, object_id): expires=600, acks_late=False, autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 3, 'countdown': 10}, + retry_kwargs={'max_retries': 2, 'countdown': 10}, retry_backoff=True, retry_backoff_max=700, retry_jitter=True) @@ -137,7 +134,7 @@ def delete_orphaned_document_files(self): expires=600, acks_late=False, autoretry_for=(Exception, ), - retry_kwargs={'max_retries': 3, 'countdown': 10}, + retry_kwargs={'max_retries': 2, 'countdown': 10}, retry_backoff=True, retry_backoff_max=700, retry_jitter=True) diff --git a/geonode/documents/templates/documents/document_detail.html b/geonode/documents/templates/documents/document_detail.html index 9279309da05..570de9325b1 100644 --- a/geonode/documents/templates/documents/document_detail.html +++ b/geonode/documents/templates/documents/document_detail.html @@ -32,7 +32,7 @@

{{ resource.title }}

{% if "download_resourcebase" in perms_list %} - {% if resource.extension|lower in audiotypes and resource.doc_file %} + {% if resource.extension|lower in audiotypes and resource.files %} {% get_mime_type mimetypemap resource.extension as mimetype %} - {% elif resource.extension|lower in imgtypes and resource.doc_file %} + {% elif resource.extension|lower in imgtypes and resource.files %} - {% elif resource.extension|lower in videotypes and resource.doc_file %} + {% elif resource.extension|lower in videotypes and resource.files %} {% get_mime_type mimetypemap resource.extension as mimetype %} - {% elif resource.doc_file %} + {% elif resource.files %}

{% trans "Download the" %} {{ resource }} {% trans "document" %}

{% elif resource.doc_url %}

{% trans "Download the" %} {{ resource }} {% trans "document" %}. ({% trans 'External Resource' %})

@@ -128,9 +128,9 @@

{% trans 'Average Rating' %}

  • {% if "download_resourcebase" in perms_list %} - {% if resource.extension|lower in imgtypes and resource.doc_file %} + {% if resource.extension|lower in imgtypes and resource.files %} - {% elif resource.doc_file %} + {% elif resource.files %} {% elif resource.doc_url %} @@ -186,6 +186,9 @@

    {% trans "Thumbnail" %}

    {% trans "Document" %}

    {% if "change_resourcebase" in perms_list %} + {% trans "Clone" %} + {% endif %} + {% if "change_resourcebase" in perms_list %} {% trans "Replace" %} {% endif %} {% if "delete_resourcebase" in perms_list %} @@ -289,4 +292,62 @@

    {% trans "Permissions" %}

    {% if FAVORITE_ENABLED %} {% include "favorite/_favorite_js.html" %} {% endif %} + + + {% endblock extra_script %} diff --git a/geonode/documents/templates/documents/document_list.html b/geonode/documents/templates/documents/document_list.html index 20c0b2b553e..a3825edd90b 100644 --- a/geonode/documents/templates/documents/document_list.html +++ b/geonode/documents/templates/documents/document_list.html @@ -15,7 +15,7 @@

    {% trans "Explore Documents" %}

{% with include_type_filter='true' %} {% with header='Document Type' %} - {% with filter='doc_type__in' %} + {% with filter='storetype__in' %} {% include "search/_search_content.html" %} {% endwith %} {% endwith %} diff --git a/geonode/documents/templates/documents/document_metadata_advanced.html b/geonode/documents/templates/documents/document_metadata_advanced.html index 554c1d2b616..6dc349f8ebb 100644 --- a/geonode/documents/templates/documents/document_metadata_advanced.html +++ b/geonode/documents/templates/documents/document_metadata_advanced.html @@ -4,6 +4,7 @@ {% load base_tags %} {% load bootstrap_tags %} {% load guardian_tags %} +{% load client_lib_tags %} {% block title %}{{ document.title }} — {{ block.super }}{% endblock %} @@ -26,7 +27,7 @@
- {% trans "Explore Documents" %} + {% trans "Explore Documents" %}

{% trans "Edit Metadata" %}

@@ -59,7 +60,7 @@

{% trans "Edit Metadata" %}

{% endif %}
- {% trans "Return to Document" %} + {% trans "Return to Document" %}
{% csrf_token %} @@ -132,7 +133,7 @@

{% trans "Metadata Provider" %}

- {% trans "Return to Document" %} + {% trans "Return to Document" %}
diff --git a/geonode/documents/templates/documents/document_replace.html b/geonode/documents/templates/documents/document_replace.html index 444a93b46b4..9f974913cf0 100644 --- a/geonode/documents/templates/documents/document_replace.html +++ b/geonode/documents/templates/documents/document_replace.html @@ -17,7 +17,7 @@

{% trans "Replace " %}{{ document.title }}

{{ form.errors }} - {% if document.doc_file %} + {% if document.files %} {{ form.doc_file }} {% elif document.doc_url %} {{ form.doc_url }} diff --git a/geonode/documents/templates/documents/document_thumb_upload.html b/geonode/documents/templates/documents/document_thumb_upload.html index cefdc90b420..c0b910f3837 100644 --- a/geonode/documents/templates/documents/document_thumb_upload.html +++ b/geonode/documents/templates/documents/document_thumb_upload.html @@ -2,6 +2,7 @@ {% load static %} {% load bootstrap_tags %} {% load i18n %} +{% load client_lib_tags %} {% block title %} {% trans "Upload Document's Thumbnail" %} - {{ block.super }} {% endblock %} @@ -9,7 +10,7 @@ {% block body_outer %}
- {% trans "Explore Documents" %} + {% trans "Explore Documents" %}

{% trans "Upload Document's Thumbnail" %}

diff --git a/geonode/documents/templates/documents/document_upload.html b/geonode/documents/templates/documents/document_upload.html index 5d3922f1944..2b18710aaf0 100644 --- a/geonode/documents/templates/documents/document_upload.html +++ b/geonode/documents/templates/documents/document_upload.html @@ -43,6 +43,11 @@

{% trans "Permissions" %}

$('#id_title').val($('#id_doc_file').val().replace("C:\\fakepath\\", "")); } }); + $('#id_doc_url').on('change', function(){ + if($('#id_title').val() == ''){ + $('#id_title').val($('#id_doc_url').val()); + } + }); $("#id_links").select2({ width: '100%' }); diff --git a/geonode/documents/templates/documents/document_upload_base.html b/geonode/documents/templates/documents/document_upload_base.html index 9d729b93229..a7f3198015a 100644 --- a/geonode/documents/templates/documents/document_upload_base.html +++ b/geonode/documents/templates/documents/document_upload_base.html @@ -3,6 +3,7 @@ {% load base_tags %} {% load guardian_tags %} {% load pagination_tags %} +{% load client_lib_tags %} {% block title %} {{ block.super }} {% endblock %} @@ -10,7 +11,7 @@ {% block body_outer %}
- {% trans "Explore Documents" %} + {% trans "Explore Documents" %}

{% trans "Upload Documents" %}

{% trans "Allowed document types" %}:

diff --git a/geonode/documents/templatetags/__init__.py b/geonode/documents/templatetags/__init__.py index fe4e643c905..0044807c781 100644 --- a/geonode/documents/templatetags/__init__.py +++ b/geonode/documents/templatetags/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/documents/templatetags/document_tags.py b/geonode/documents/templatetags/document_tags.py index e149e70f908..034df799e9c 100644 --- a/geonode/documents/templatetags/document_tags.py +++ b/geonode/documents/templatetags/document_tags.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index ea57355fcba..104b71a26d0 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -23,44 +22,57 @@ when you run "manage.py test". """ -from geonode.tests.base import GeoNodeBaseTestSupport - import os import io import json - import gisdata + +from unittest.mock import patch + from django.urls import reverse from django.conf import settings from django.contrib.auth.models import Group from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.core.files.uploadedfile import SimpleUploadedFile -from django.core.files.storage import default_storage as storage from guardian.shortcuts import get_perms, get_anonymous_user -from .forms import DocumentCreateForm - -from geonode.groups.models import ( - GroupProfile, - GroupMember) from geonode.maps.models import Map from geonode.layers.models import Layer from geonode.compat import ensure_string from geonode.base.thumb_utils import get_thumbs from geonode.base.models import License, Region from geonode.documents import DocumentsAppConfig +from geonode.storage.manager import storage_manager +from geonode.resource.manager import resource_manager from geonode.documents.forms import DocumentFormMixin +from geonode.tests.base import GeoNodeBaseTestSupport from geonode.tests.utils import NotificationsTestsHelper -from geonode.base.populate_test_data import create_models from geonode.documents.enumerations import DOCUMENT_TYPE_MAP from geonode.documents.models import Document, DocumentResourceLink +from geonode.groups.models import ( + GroupProfile, + GroupMember) +from geonode.base.populate_test_data import ( + all_public, + create_models, + remove_models) + +from .forms import DocumentCreateForm + class DocumentsTest(GeoNodeBaseTestSupport): type = 'document' + + fixtures = [ + 'initial_data.json', + 'group_test_data.json', + 'default_oauth_apps.json' + ] + perm_spec = { "users": { "admin": [ @@ -69,8 +81,19 @@ class DocumentsTest(GeoNodeBaseTestSupport): "view_resourcebase"]}, "groups": {}} + @classmethod + def setUpClass(cls): + super().setUpClass() + create_models(type=cls.get_type, integration=cls.get_integration) + all_public() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration) + def setUp(self): - super(DocumentsTest, self).setUp() + super().setUp() create_models('map') self.imgfile = io.BytesIO( b'GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00' @@ -94,33 +117,30 @@ def test_document_mimetypes_rendering(self): self.assertTrue('tiff' in ARCHIVETYPES) self.assertTrue('pbm' in ARCHIVETYPES) - def test_create_document_with_no_rel(self): + @patch("geonode.documents.tasks.create_document_thumbnail") + def test_create_document_with_no_rel(self, thumb): """Tests the creation of a document with no relations""" - - f = SimpleUploadedFile( - 'test_img_file.gif', - self.imgfile.read(), - 'image/gif') + thumb.return_value = True + f = [f"{settings.MEDIA_ROOT}/img.gif"] superuser = get_user_model().objects.get(pk=2) c = Document.objects.create( - doc_file=f, + files=f, owner=superuser, title='theimg') c.set_default_permissions() self.assertEqual(Document.objects.get(pk=c.id).title, 'theimg') - def test_create_document_with_rel(self): + @patch("geonode.documents.tasks.create_document_thumbnail") + def test_create_document_with_rel(self, thumb): """Tests the creation of a document with no a map related""" - f = SimpleUploadedFile( - 'test_img_file.gif', - self.imgfile.read(), - 'image/gif') + thumb.return_value = True + f = [f"{settings.MEDIA_ROOT}/img.gif"] superuser = get_user_model().objects.get(pk=2) c = Document.objects.create( - doc_file=f, + files=f, owner=superuser, title='theimg') @@ -138,10 +158,14 @@ def test_create_document_url(self): """Tests creating an external document instead of a file.""" superuser = get_user_model().objects.get(pk=2) - c = Document.objects.create(doc_url="http://geonode.org/map.pdf", - owner=superuser, - title="GeoNode Map", - ) + c = resource_manager.create( + None, + resource_type=Document, + defaults=dict( + doc_url="http://geonode.org/map.pdf", + owner=superuser, + title="GeoNode Map", + )) doc = Document.objects.get(pk=c.id) self.assertEqual(doc.title, "GeoNode Map") self.assertEqual(doc.extension, "pdf") @@ -237,7 +261,9 @@ def test_document_details(self): response = self.client.get(reverse('document_detail', args=(str(d.id),))) self.assertEqual(response.status_code, 200) - def test_document_metadata_details(self): + @patch("geonode.documents.tasks.create_document_thumbnail") + def test_document_metadata_details(self, thumb): + thumb.return_value = True d = Document.objects.all().first() d.set_default_permissions() @@ -293,7 +319,7 @@ def test_set_document_permissions(self): """Verify that the set_document_permissions view is behaving as expected """ # Get a document to work with - document = Document.objects.all()[0] + document = Document.objects.first() # Set the Permissions document.set_permissions(self.perm_spec) @@ -315,22 +341,23 @@ def test_set_document_permissions(self): user = get_user_model().objects.get(username=username) self.assertTrue(user.has_perm(perm, document.get_self_resource())) - def test_ajax_document_permissions(self): + @patch("geonode.documents.tasks.create_document_thumbnail") + def test_ajax_document_permissions(self, create_thumb): """Verify that the ajax_document_permissions view is behaving as expected """ - + create_thumb.return_value = True # Setup some document names to work with - f = SimpleUploadedFile( - 'test_img_file.gif', - self.imgfile.read(), - 'image/gif') + f = [f"{settings.MEDIA_ROOT}/img.gif"] superuser = get_user_model().objects.get(pk=2) - document = Document.objects.create( - doc_file=f, - owner=superuser, - title='theimg') - document.set_default_permissions() + document = resource_manager.create( + None, + resource_type=Document, + defaults=dict( + files=f, + owner=superuser, + title='theimg', + is_approved=True)) document_id = document.id invalid_document_id = 20 @@ -340,7 +367,7 @@ def test_ajax_document_permissions(self): 'resource_permissions', args=( invalid_document_id,)), data=json.dumps( self.perm_spec), content_type="application/json") - self.assertEqual(response.status_code, 404) + self.assertEqual(response.status_code, 401) # Test that GET returns permissions response = self.client.get(reverse('resource_permissions', args=(document_id,))) @@ -453,11 +480,12 @@ def test_batch_edit(self): class DocumentModerationTestCase(GeoNodeBaseTestSupport): def setUp(self): - super(DocumentModerationTestCase, self).setUp() + super().setUp() self.user = 'admin' self.passwd = 'admin' create_models(type=b'document') create_models(type=b'map') + self.project_root = os.path.abspath(os.path.dirname(__file__)) self.document_upload_url = f"{(reverse('document_upload'))}?no__redirect=true" self.u = get_user_model().objects.get(username=self.user) self.u.email = 'test@email.com' @@ -471,19 +499,20 @@ def _get_input_path(self): def test_document_upload_redirect(self): with self.settings(ADMIN_MODERATE_UPLOADS=False): self.client.login(username=self.user, password=self.passwd) - input_path = self._get_input_path() - document_upload_url = str(reverse('document_upload')) - with open(input_path, 'rb') as f: - data = {'title': 'document title', - 'doc_file': f, - 'resource': '', - 'extension': 'txt', - 'permissions': '{}', - } - resp = self.client.post(document_upload_url, data=data) - if resp.status_code == 200: - content = resp.content.decode('utf-8') - self.asserTrue("document title" in content) + dname = 'document title' + with open(os.path.join(f"{self.project_root}", "tests/data/img.gif"), "rb") as f: + data = { + 'title': dname, + 'doc_file': f, + 'resource': '', + 'extension': 'gif', + 'permissions': '{}', + } + resp = self.client.post(self.document_upload_url, data=data) + self.assertEqual(resp.status_code, 200, resp.content) + content = json.loads(resp.content.decode('utf-8')) + self.assertTrue(content["success"]) + self.assertIn("url", content) def test_moderated_upload(self): """ @@ -491,19 +520,18 @@ def test_moderated_upload(self): """ with self.settings(ADMIN_MODERATE_UPLOADS=False): self.client.login(username=self.user, password=self.passwd) - input_path = self._get_input_path() - - with open(input_path, 'rb') as f: - data = {'title': 'document title', - 'doc_file': f, - 'resource': '', - 'extension': 'txt', - 'permissions': '{}', - } - resp = self.client.post(self.document_upload_url, data=data) - self.assertEqual(resp.status_code, 200) dname = 'document title' + with open(os.path.join(f"{self.project_root}", "tests/data/img.gif"), "rb") as f: + data = { + 'title': dname, + 'doc_file': f, + 'resource': '', + 'extension': 'txt', + 'permissions': '{}', + } + resp = self.client.post(self.document_upload_url, data=data) + self.assertEqual(resp.status_code, 200, resp.content) _d = Document.objects.get(title=dname) self.assertTrue(_d.is_published) @@ -511,19 +539,18 @@ def test_moderated_upload(self): _d.delete() from geonode.documents.utils import delete_orphaned_document_files - _, document_files_before = storage.listdir( - os.path.join("documents", "document")) - deleted = delete_orphaned_document_files() - _, document_files_after = storage.listdir( - os.path.join("documents", "document")) - self.assertTrue(len(deleted) > 0) - self.assertEqual(set(deleted), set(document_files_before) - set(document_files_after)) + if storage_manager.exists(os.path.join("documents", "document")): + _, document_files_before = storage_manager.listdir(os.path.join("documents", "document")) + deleted = delete_orphaned_document_files() + _, document_files_after = storage_manager.listdir(os.path.join("documents", "document")) + self.assertTrue(len(deleted) > 0) + self.assertEqual(set(deleted), set(document_files_before) - set(document_files_after)) from geonode.base.utils import delete_orphaned_thumbs thumb_files_before = get_thumbs() deleted = delete_orphaned_thumbs() thumb_files_after = get_thumbs() - if len(thumb_files_before): + if deleted: self.assertTrue( len(deleted) > 0, f"before: {thumb_files_before} - deleted: {deleted} - after: {thumb_files_after}") @@ -533,27 +560,28 @@ def test_moderated_upload(self): fn = os.path.join( os.path.join("documents", "document"), os.path.basename(input_path)) - self.assertFalse(storage.exists(fn)) + self.assertFalse(storage_manager.exists(fn)) files = [thumb for thumb in get_thumbs() if uuid in thumb] - self.assertEqual(len(files), 0) + if files and len(files): + self.assertEqual(len(files), 1) with self.settings(ADMIN_MODERATE_UPLOADS=True): self.client.login(username=self.user, password=self.passwd) - norman = get_user_model().objects.get(username="norman") group = GroupProfile.objects.get(slug="bar") input_path = self._get_input_path() + dname = 'document title' with open(input_path, 'rb') as f: - data = {'title': 'document title', - 'doc_file': f, - 'resource': '', - 'extension': 'txt', - 'permissions': '{}', - } + data = { + 'title': dname, + 'doc_file': f, + 'resource': '', + 'extension': 'txt', + 'permissions': '{}', + } resp = self.client.post(self.document_upload_url, data=data) self.assertEqual(resp.status_code, 200) - dname = 'document title' _d = Document.objects.get(title=dname) self.assertFalse(_d.is_approved) self.assertTrue(_d.is_published) @@ -564,14 +592,12 @@ def test_moderated_upload(self): self.assertTrue(group.user_is_role(norman, "manager")) self.client.login(username="norman", password="norman") - resp = self.client.get( - reverse('document_detail', args=(_d.id,))) + resp = self.client.get(reverse('document_detail', args=(_d.id,))) # Forbidden self.assertEqual(resp.status_code, 403) _d.group = group.group _d.save() - resp = self.client.get( - reverse('document_detail', args=(_d.id,))) + resp = self.client.get(reverse('document_detail', args=(_d.id,))) # Allowed - edit permissions self.assertEqual(resp.status_code, 200) perms_list = get_perms(norman, _d.get_self_resource()) + get_perms(norman, _d) @@ -664,16 +690,11 @@ def setUp(self): def test_create_document_with_links(self): """Tests the creation of document links.""" - f = SimpleUploadedFile( - 'test_img_file.gif', - self.test_file.read(), - 'image/gif' - ) - + f = [f"{settings.MEDIA_ROOT}/img.gif"] superuser = get_user_model().objects.get(pk=2) d = Document.objects.create( - doc_file=f, + files=f, owner=superuser, title='theimg' ) @@ -731,6 +752,7 @@ def test_create_document_with_links(self): class DocumentViewTestCase(GeoNodeBaseTestSupport): + fixtures = [ 'initial_data.json', 'group_test_data.json', @@ -741,16 +763,17 @@ def setUp(self): self.not_admin = get_user_model().objects.create(username='r-lukaku', is_active=True) self.not_admin.set_password('very-secret') self.not_admin.save() - self.imgfile = io.BytesIO( - b'GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00' - b'\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;') - f = SimpleUploadedFile( - 'test_img_file.gif', - self.imgfile.read(), - 'image/gif') - self.test_doc = Document.objects.create(doc_file=f, owner=self.not_admin, title='test', is_approved=True) + self.files = [f"{settings.MEDIA_ROOT}/img.gif"] + self.test_doc = resource_manager.create( + None, + resource_type=Document, + defaults=dict( + files=self.files, + owner=self.not_admin, + title='test', + is_approved=True)) self.perm_spec = {"users": {"AnonymousUser": []}} - self.dock_link_url = reverse('document_link', args=(self.test_doc.pk,)) + self.doc_link_url = reverse('document_link', args=(self.test_doc.pk,)) def test_that_keyword_multiselect_is_disabled_for_non_admin_users(self): """ @@ -830,9 +853,9 @@ def test_that_non_admin_user_can_create_edit_keyword_when_freetext_keywords_read def test_document_link_with_permissions(self): self.test_doc.set_permissions(self.perm_spec) # Get link as Anonymous user - response = self.client.get(self.dock_link_url) + response = self.client.get(self.doc_link_url) self.assertEqual(response.status_code, 401) # Access resource with user logged-in self.client.login(username=self.not_admin.username, password='very-secret') - response = self.client.get(self.dock_link_url) - self.assertEqual(response.status_code, 200) + response = self.client.get(self.doc_link_url) + self.assertEqual(response.status_code, 404) diff --git a/geonode/documents/tests/data/img.gif b/geonode/documents/tests/data/img.gif new file mode 100644 index 00000000000..56959b6411a Binary files /dev/null and b/geonode/documents/tests/data/img.gif differ diff --git a/geonode/documents/translation.py b/geonode/documents/translation.py index 780512b717c..96fa3879218 100644 --- a/geonode/documents/translation.py +++ b/geonode/documents/translation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/documents/urls.py b/geonode/documents/urls.py index 6c15c026ee1..5eda95bf594 100644 --- a/geonode/documents/urls.py +++ b/geonode/documents/urls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/documents/utils.py b/geonode/documents/utils.py index 73381897420..9ed1099b683 100644 --- a/geonode/documents/utils.py +++ b/geonode/documents/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -24,9 +23,8 @@ # Standard Modules import os import logging - +from geonode.storage.manager import storage_manager # Django functionality -from django.core.files.storage import default_storage as storage from django.http import HttpResponse from django.shortcuts import get_object_or_404 from django.template import loader @@ -47,13 +45,13 @@ def delete_orphaned_document_files(): Deletes orphaned files of deleted documents. """ deleted = [] - _, files = storage.listdir(os.path.join("documents", "document")) + _, files = storage_manager.listdir(os.path.join("documents", "document")) for filename in files: if Document.objects.filter(doc_file__contains=filename).count() == 0: logger.debug(f"Deleting orphaned document {filename}") try: - storage.delete(os.path.join( + storage_manager.delete(os.path.join( os.path.join("documents", "document"), filename)) deleted.append(filename) except NotImplementedError as e: @@ -80,8 +78,14 @@ def get_download_response(request, docid, attachment=False): if attachment: register_event(request, EventType.EVENT_DOWNLOAD, document) filename = slugify(os.path.splitext(os.path.basename(document.title))[0]) - return DownloadResponse( - document.doc_file, - basename=f'{filename}.{document.extension}', - attachment=attachment + + if document.files and storage_manager.exists(document.files[0]): + return DownloadResponse( + storage_manager.open(document.files[0]).file, + basename=f'{filename}.{document.extension}', + attachment=attachment + ) + return HttpResponse( + "File is not available", + status=404 ) diff --git a/geonode/documents/views.py b/geonode/documents/views.py index 0baa8741138..80f3a671ee0 100644 --- a/geonode/documents/views.py +++ b/geonode/documents/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -17,48 +16,63 @@ # along with this program. If not, see . # ######################################################################### +import os import json +import shutil import logging -import traceback -from itertools import chain +import tempfile import warnings +import traceback +from itertools import chain +from dal import autocomplete from guardian.shortcuts import get_objects_for_user +from django.db.models import F +from django.urls import reverse +from django.conf import settings +from django.contrib import messages from django.shortcuts import render -from django.http import HttpResponse, HttpResponseRedirect, Http404 +from django.forms.utils import ErrorList from django.utils.translation import ugettext as _ from django.contrib.auth.decorators import login_required -from django.conf import settings -from django.urls import reverse -from django.core.exceptions import PermissionDenied, ObjectDoesNotExist from django.views.generic.edit import UpdateView, CreateView -from django.db.models import F -from django.forms.utils import ErrorList +from django.http import HttpResponse, HttpResponseRedirect, Http404 +from django.core.exceptions import PermissionDenied, ObjectDoesNotExist -from geonode.base.utils import ManageResourceOwnerPermissions -from geonode.decorators import check_keyword_write_perms -from geonode.documents.utils import get_download_response from geonode.utils import resolve_object -from geonode.security.views import _perms_info_json +from geonode.base.views import batch_modify +from geonode.utils import build_social_links from geonode.people.forms import ProfileForm -from geonode.base.auth import get_or_create_token +from geonode.monitoring import register_event from geonode.base.bbox_utils import BBOXHelper -from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm -from geonode.base.models import ( - Thesaurus, - TopicCategory) -from geonode.documents.enumerations import DOCUMENT_TYPE_MAP, DOCUMENT_MIMETYPE_MAP -from geonode.documents.models import Document, get_related_resources -from geonode.documents.forms import DocumentForm, DocumentCreateForm, DocumentReplaceForm -from geonode.utils import build_social_links from geonode.groups.models import GroupProfile -from geonode.base.views import batch_modify -from geonode.monitoring import register_event from geonode.monitoring.models import EventType +from geonode.base.auth import get_or_create_token +from geonode.security.views import _perms_info_json +from geonode.storage.manager import storage_manager +from geonode.resource.manager import resource_manager +from geonode.resource.utils import get_related_resources +from geonode.decorators import check_keyword_write_perms from geonode.security.utils import get_visible_resources +from geonode.base.utils import ManageResourceOwnerPermissions +from geonode.base.forms import ( + CategoryForm, + TKeywordForm, + ThesaurusAvailableForm) +from geonode.base.models import ( + Thesaurus, + TopicCategory) -from dal import autocomplete +from .utils import get_download_response +from .enumerations import ( + DOCUMENT_TYPE_MAP, + DOCUMENT_MIMETYPE_MAP) +from .models import Document +from .forms import ( + DocumentForm, + DocumentCreateForm, + DocumentReplaceForm) logger = logging.getLogger("geonode.documents.views") @@ -167,7 +181,7 @@ def document_detail(request, docid): if exif: context_dict['exif_data'] = exif except Exception: - logger.error("Exif extraction failed.") + logger.debug("Exif extraction failed.") if request.user.is_authenticated: if getattr(settings, 'FAVORITE_ENABLED', False): @@ -197,14 +211,15 @@ class DocumentUploadView(CreateView): form_class = DocumentCreateForm def get_context_data(self, **kwargs): - context = super(DocumentUploadView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) context['ALLOWED_DOC_TYPES'] = ALLOWED_DOC_TYPES return context def form_invalid(self, form): + messages.error(self.request, f"{form.errors}") if self.request.GET.get('no__redirect', False): out = {'success': False} - out['message'] = "" + out['message'] = f"{form.errors}" status_code = 400 return HttpResponse( json.dumps(out), @@ -215,22 +230,50 @@ def form_invalid(self, form): form.title = None form.doc_file = None form.doc_url = None - return self.render_to_response(self.get_context_data(form=form)) + return self.render_to_response( + self.get_context_data(request=self.request, form=form)) def form_valid(self, form): """ If the form is valid, save the associated model. """ - self.object = form.save(commit=False) - self.object.owner = self.request.user + doc_form = form.cleaned_data + + file = doc_form.pop('doc_file', None) + if file: + tempdir = tempfile.mkdtemp(dir=settings.STATIC_ROOT) + dirname = os.path.basename(tempdir) + filepath = storage_manager.save(f"{dirname}/{file.name}", file) + storage_path = storage_manager.path(filepath) + self.object = resource_manager.create( + None, + resource_type=Document, + defaults=dict( + owner=self.request.user, + doc_url=doc_form.pop('doc_url', None), + title=doc_form.pop('title', file.name), + files=[storage_path]) + ) + if tempdir != os.path.dirname(storage_path): + shutil.rmtree(tempdir, ignore_errors=True) + else: + self.object = resource_manager.create( + None, + resource_type=Document, + defaults=dict( + owner=self.request.user, + doc_url=doc_form.pop('doc_url', None), + title=doc_form.pop('title', None)) + ) if settings.ADMIN_MODERATE_UPLOADS: self.object.is_approved = False if settings.RESOURCE_PUBLISHING: self.object.is_published = False - self.object.save() - form.save_many2many() - self.object.set_permissions(form.cleaned_data['permissions']) + + resource_manager.set_permissions( + None, instance=self.object, permissions=form.cleaned_data["permissions"], created=True + ) abstract = None date = None @@ -250,26 +293,22 @@ def form_valid(self, form): bbox = exif_metadata.get('bbox', None) abstract = exif_metadata.get('abstract', None) except Exception: - logger.error("Exif extraction failed.") - - if abstract: - self.object.abstract = abstract - - if date: - self.object.date = date - self.object.date_type = "Creation" - - if len(regions) > 0: - self.object.regions.add(*regions) + logger.debug("Exif extraction failed.") + + resource_manager.update( + self.object.uuid, + instance=self.object, + keywords=keywords, + regions=regions, + vals=dict( + abstract=abstract, + date=date, + date_type="Creation", + bbox_polygon=BBOXHelper.from_xy(bbox).as_polygon() if bbox else None + ), + notify=True) + resource_manager.set_thumbnail(self.object.uuid, instance=self.object, overwrite=False) - if len(keywords) > 0: - self.object.keywords.add(*keywords) - - if bbox: - bbox = BBOXHelper.from_xy(bbox) - self.object.bbox_polygon = bbox.as_polygon() - - self.object.save(notify=True) register_event(self.request, EventType.EVENT_UPLOAD, self.object) if self.request.GET.get('no__redirect', False): @@ -304,7 +343,7 @@ class DocumentUpdateView(UpdateView): context_object_name = 'document' def get_context_data(self, **kwargs): - context = super(DocumentUpdateView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) context['ALLOWED_DOC_TYPES'] = ALLOWED_DOC_TYPES return context @@ -312,8 +351,16 @@ def form_valid(self, form): """ If the form is valid, save the associated model. """ - self.object = form.save() + self.object = resource_manager.replace( + self.object, + vals={ + 'files': form.cleaned_data.get('doc_file'), + 'doc_url': form.cleaned_data.get('doc_url'), + 'user': self.request.user + }) + register_event(self.request, EventType.EVENT_CHANGE, self.object) + return HttpResponseRedirect( reverse( 'document_detail', @@ -453,15 +500,18 @@ def document_metadata( new_author = author_form.save() document = document_form.instance - if new_poc is not None and new_author is not None: - document.poc = new_poc - document.metadata_author = new_author - document.keywords.clear() - document.keywords.add(*new_keywords) - document.regions.clear() - document.regions.add(*new_regions) - document.category = new_category - document.save(notify=True) + resource_manager.update( + document.uuid, + instance=document, + keywords=new_keywords, + regions=new_regions, + vals=dict( + poc=new_poc or document.poc, + metadata_author=new_author or document.metadata_author, + category=new_category + ), + notify=True) + resource_manager.set_thumbnail(document.uuid, instance=document, overwrite=False) document_form.save_many2many() register_event(request, EventType.EVENT_CHANGE_METADATA, document) @@ -604,7 +654,8 @@ def document_remove(request, docid, template='documents/document_remove.html'): "document": document }) if request.method == 'POST': - document.delete() + resource_manager.delete(document.uuid, instance=document) + register_event(request, EventType.EVENT_REMOVE, document) return HttpResponseRedirect(reverse("document_browse")) else: diff --git a/geonode/favorite/__init__.py b/geonode/favorite/__init__.py index b0fb2f81c70..79177e00bdd 100644 --- a/geonode/favorite/__init__.py +++ b/geonode/favorite/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/favorite/migrations/0001_initial.py b/geonode/favorite/migrations/0001_initial.py index d72ac520a3a..e6228722dcc 100644 --- a/geonode/favorite/migrations/0001_initial.py +++ b/geonode/favorite/migrations/0001_initial.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-23 10:36 @@ -33,6 +32,6 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='favorite', - unique_together=set([('user', 'content_type', 'object_id')]), + unique_together={('user', 'content_type', 'object_id')}, ), ] diff --git a/geonode/favorite/models.py b/geonode/favorite/models.py index b2d6e34811b..1c3e7fa9f32 100644 --- a/geonode/favorite/models.py +++ b/geonode/favorite/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/favorite/tests.py b/geonode/favorite/tests.py index 6e69f153517..0f27866cea8 100644 --- a/geonode/favorite/tests.py +++ b/geonode/favorite/tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -42,7 +41,7 @@ class FavoriteTest(GeoNodeBaseTestSupport): """ def setUp(self): - super(FavoriteTest, self).setUp() + super().setUp() self.adm_un = "admin" self.adm_pw = "admin" diff --git a/geonode/favorite/urls.py b/geonode/favorite/urls.py index f891bef0821..fdf193711cd 100644 --- a/geonode/favorite/urls.py +++ b/geonode/favorite/urls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/favorite/utils.py b/geonode/favorite/utils.py index e09f952d778..985b1ed0d7c 100644 --- a/geonode/favorite/utils.py +++ b/geonode/favorite/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/favorite/views.py b/geonode/favorite/views.py index 6ff3a156c5b..36704a7272d 100644 --- a/geonode/favorite/views.py +++ b/geonode/favorite/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo diff --git a/geonode/geoapps/__init__.py b/geonode/geoapps/__init__.py index dc685246de8..a789f94d1b7 100644 --- a/geonode/geoapps/__init__.py +++ b/geonode/geoapps/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/geoapps/admin.py b/geonode/geoapps/admin.py index 988def4d103..ebff453a0d2 100644 --- a/geonode/geoapps/admin.py +++ b/geonode/geoapps/admin.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo @@ -22,14 +21,10 @@ from modeltranslation.admin import TabbedTranslationAdmin -from geonode.geoapps.models import GeoApp, GeoAppData +from geonode.geoapps.models import GeoApp from geonode.base.admin import ResourceBaseAdminForm -class GeoAppDataInline(admin.TabularInline): - model = GeoAppData - - class GeoAppAdminForm(ResourceBaseAdminForm): class Meta(ResourceBaseAdminForm.Meta): @@ -38,7 +33,6 @@ class Meta(ResourceBaseAdminForm.Meta): class GeoAppAdmin(TabbedTranslationAdmin): - inlines = [GeoAppDataInline, ] list_display_links = ('title',) list_display = ('id', 'title', 'type', 'owner', 'category', 'group', 'is_approved', 'is_published',) list_editable = ('owner', 'category', 'group', 'is_approved', 'is_published',) diff --git a/geonode/geoapps/api/__init__.py b/geonode/geoapps/api/__init__.py index fe4e643c905..0044807c781 100644 --- a/geonode/geoapps/api/__init__.py +++ b/geonode/geoapps/api/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/geoapps/api/permissions.py b/geonode/geoapps/api/permissions.py index eb16decd377..22ee01ba5a3 100644 --- a/geonode/geoapps/api/permissions.py +++ b/geonode/geoapps/api/permissions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/geoapps/api/serializers.py b/geonode/geoapps/api/serializers.py index 0262a6ba7a5..9ee57514453 100644 --- a/geonode/geoapps/api/serializers.py +++ b/geonode/geoapps/api/serializers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo @@ -17,60 +16,24 @@ # along with this program. If not, see . # ######################################################################### -import json import logging -from django.contrib.auth import get_user_model -from dynamic_rest.fields.fields import DynamicRelationField -from dynamic_rest.serializers import DynamicModelSerializer -from geonode.base.api.serializers import ResourceBaseSerializer -from geonode.geoapps.models import GeoApp, GeoAppData from rest_framework.serializers import ValidationError -logger = logging.getLogger(__name__) - - -class GeoAppDataField(DynamicRelationField): - - def value_to_string(self, obj): - value = self.value_from_object(obj) - return self.get_prep_value(value) - - -class GeoAppDataSerializer(DynamicModelSerializer): - - class Meta: - ref_name = 'GeoAppData' - model = GeoAppData - name = 'GeoAppData' - fields = ('pk', 'blob') +from django.contrib.auth import get_user_model - def to_internal_value(self, data): - return data +from geonode.geoapps.models import GeoApp +from geonode.resource.manager import resource_manager +from geonode.base.api.serializers import ResourceBaseSerializer - def to_representation(self, value): - data = GeoAppData.objects.filter(resource__id=value).first() - if data and data.blob: - if isinstance(data.blob, dict): - return data.blob - return json.loads(data.blob) - return {} +logger = logging.getLogger(__name__) class GeoAppSerializer(ResourceBaseSerializer): - """ - - Deferred / not Embedded --> ?include[]=data - """ - data = GeoAppDataField( - GeoAppDataSerializer, - source='id', - many=False, - embed=False, - deferred=True) def __init__(self, *args, **kwargs): # Instantiate the superclass normally - super(GeoAppSerializer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class Meta: model = GeoApp @@ -82,16 +45,6 @@ class Meta: 'urlsuffix', 'data' ) - def to_internal_value(self, data): - if isinstance(data, str): - data = json.loads(data) - if 'data' in data: - _data = data.pop('data') - if self.is_valid(): - data['blob'] = _data - - return data - def extra_update_checks(self, validated_data): _user_profiles = {} for _key, _value in validated_data.items(): @@ -115,34 +68,67 @@ def extra_create_checks(self, validated_data): self.extra_update_checks(validated_data) + def validate(self, data): + request = self.context.get('request') + if request: + data['owner'] = request.user + return data + def create(self, validated_data): + # Sanity checks + if 'name' not in validated_data or \ + 'owner' not in validated_data: + raise ValidationError("No valid data: 'name' and 'owner' are mandatory fields!") + + if self.Meta.model.objects.filter(name=validated_data['name']).count(): + raise ValidationError("A GeoApp with the same 'name' already exists!") - # perform sanity checks - self.extra_create_checks(validated_data) + # Extract users' profiles + _user_profiles = {} + for _key, _value in validated_data.items(): + if _key in ('owner', 'poc', 'metadata_owner'): + _user_profiles[_key] = _value + for _key, _value in _user_profiles.items(): + validated_data.pop(_key) + _u = get_user_model().objects.filter(username=_value).first() + if _u: + validated_data[_key] = _u + else: + raise ValidationError(f"The specified '{_key}' does not exist!") # Extract JSON blob - _data = None + _data = {} if 'blob' in validated_data: _data = validated_data.pop('blob') # Create a new instance - _instance = self.Meta.model.objects.create(**validated_data) - - if _instance and _data: - try: - _geo_app, _created = GeoAppData.objects.get_or_create(resource=_instance) - _geo_app.blob = _data - _geo_app.save() - except Exception as e: - raise ValidationError(e) - - _instance.save() - return _instance + _instance = resource_manager.create( + None, + resource_type=self.Meta.model, + defaults=validated_data) + + return resource_manager.update( + _instance.uuid, + instance=_instance, + vals=dict( + blob=_data + ), + notify=True) def update(self, instance, validated_data): - # perform sanity checks - self.extra_update_checks(validated_data) + # Extract users' profiles + _user_profiles = {} + for _key, _value in validated_data.items(): + if _key in ('owner', 'poc', 'metadata_owner'): + _user_profiles[_key] = _value + for _key, _value in _user_profiles.items(): + validated_data.pop(_key) + _u = get_user_model().objects.filter(username=_value).first() + if _u: + validated_data[_key] = _u + else: + raise ValidationError(f"The specified '{_key}' does not exist!") # Extract JSON blob _data = None @@ -155,13 +141,6 @@ def update(self, instance, validated_data): except Exception as e: raise ValidationError(e) - if instance and _data: - try: - _geo_app, _created = GeoAppData.objects.get_or_create(resource=instance) - _geo_app.blob = _data - _geo_app.save() - except Exception as e: - raise ValidationError(e) - + instance.blob = _data instance.save() return instance diff --git a/geonode/geoapps/api/tests.py b/geonode/geoapps/api/tests.py index cb4baa67916..dda0b8792c5 100644 --- a/geonode/geoapps/api/tests.py +++ b/geonode/geoapps/api/tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2016 OSGeo @@ -18,31 +17,19 @@ # ######################################################################### import json -import django import logging from django.urls import reverse -from django.conf.urls import url, include -from django.views.generic import TemplateView from django.contrib.auth import get_user_model -from django.views.i18n import JavaScriptCatalog -from rest_framework.test import APITestCase, URLPatternsTestCase - -from geonode.api.urls import router -from geonode.services.views import services -from geonode.maps.views import map_embed -from geonode.geoapps.views import geoapp_edit -from geonode.layers.views import layer_upload, layer_embed -from geonode.geoapps.models import GeoApp, GeoAppData - -from geonode import geoserver -from geonode.utils import check_ogc_backend +from rest_framework.test import APITestCase + +from geonode.geoapps.models import GeoApp from geonode.base.populate_test_data import create_models logger = logging.getLogger(__name__) -class BaseApiTests(APITestCase, URLPatternsTestCase): +class GeoAppsApiTests(APITestCase): fixtures = [ 'initial_data.json', @@ -50,66 +37,6 @@ class BaseApiTests(APITestCase, URLPatternsTestCase): 'default_oauth_apps.json' ] - urlpatterns = [ - url(r'^home/$', - TemplateView.as_view(template_name='index.html'), - name='home'), - url(r'^help/$', - TemplateView.as_view(template_name='help.html'), - name='help'), - url(r'^developer/$', - TemplateView.as_view( - template_name='developer.html'), - name='developer'), - url(r'^about/$', - TemplateView.as_view(template_name='about.html'), - name='about'), - url(r'^privacy_cookies/$', - TemplateView.as_view(template_name='privacy-cookies.html'), - name='privacy-cookies'), - url(r"^account/", include("allauth.urls")), - url(r'^people/', include('geonode.people.urls')), - url(r'^api/v2/', include(router.urls)), - url(r'^api/v2/', include('geonode.api.urls')), - url(r'^api/v2/api-auth/', include('rest_framework.urls', namespace='geonode_rest_framework')), - url(r'^upload$', layer_upload, name='layer_upload'), - url(r'^$', - TemplateView.as_view(template_name='layers/layer_list.html'), - {'facet_type': 'layers', 'is_layer': True}, - name='layer_browse'), - url(r'^$', - TemplateView.as_view(template_name='maps/map_list.html'), - {'facet_type': 'maps', 'is_map': True}, - name='maps_browse'), - url(r'^$', - TemplateView.as_view(template_name='documents/document_list.html'), - {'facet_type': 'documents', 'is_document': True}, - name='document_browse'), - url(r'^$', - TemplateView.as_view(template_name='groups/group_list.html'), - name='group_list'), - url(r'^search/$', - TemplateView.as_view(template_name='search/search.html'), - name='search'), - url(r'^$', services, name='services'), - url(r'^invitations/', include( - 'geonode.invitations.urls', namespace='geonode.invitations')), - url(r'^i18n/', include(django.conf.urls.i18n), name="i18n"), - url(r'^jsi18n/$', JavaScriptCatalog.as_view(), {}, name='javascript-catalog'), - url(r'^(?P[^/]+)/embed$', map_embed, name='map_embed'), - url(r'^(?P[^/]+)/embed$', layer_embed, name='layer_embed'), - url(r'^(?P[^/]+)/embed$', geoapp_edit, {'template': 'apps/app_embed.html'}, name='geoapp_embed'), - ] - - if check_ogc_backend(geoserver.BACKEND_PACKAGE): - from geonode.geoserver.views import layer_acls, resolve_user - urlpatterns += [ - url(r'^acls/?$', layer_acls, name='layer_acls'), - url(r'^acls_dep/?$', layer_acls, name='layer_acls_dep'), - url(r'^resolve_user/?$', resolve_user, name='layer_resolve_user'), - url(r'^resolve_user_dep/?$', resolve_user, name='layer_resolve_user_dep'), - ] - def setUp(self): create_models(b'document') create_models(b'map') @@ -119,11 +46,8 @@ def setUp(self): self.norman = get_user_model().objects.get(username='norman') self.gep_app = GeoApp.objects.create( title="Test GeoApp", - owner=self.bobby - ) - self.gep_app_data = GeoAppData.objects.create( - blob='{"test_data": {"test": ["test_1","test_2","test_3"]}}', - resource=self.gep_app + owner=self.bobby, + blob='{"test_data": {"test": ["test_1","test_2","test_3"]}}' ) self.gep_app.set_default_permissions() diff --git a/geonode/geoapps/api/urls.py b/geonode/geoapps/api/urls.py index 5f2034fda73..4c5588b85da 100644 --- a/geonode/geoapps/api/urls.py +++ b/geonode/geoapps/api/urls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/geoapps/api/views.py b/geonode/geoapps/api/views.py index c0f439e4460..c651f593b76 100644 --- a/geonode/geoapps/api/views.py +++ b/geonode/geoapps/api/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo @@ -47,6 +46,6 @@ class GeoAppViewSet(DynamicModelViewSet): DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, ExtentFilter, GeoAppPermissionsFilter ] - queryset = GeoApp.objects.all() + queryset = GeoApp.objects.all().order_by('-date') serializer_class = GeoAppSerializer pagination_class = GeoNodeApiPagination diff --git a/geonode/geoapps/forms.py b/geonode/geoapps/forms.py index 596035660de..cd036229217 100644 --- a/geonode/geoapps/forms.py +++ b/geonode/geoapps/forms.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo @@ -34,7 +33,7 @@ class Meta(ResourceBaseForm.Meta): ) def __init__(self, *args, **kwargs): - super(GeoAppForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) for field in self.fields: help_text = self.fields[field].help_text self.fields[field].help_text = None diff --git a/geonode/geoapps/migrations/0001_initial.py b/geonode/geoapps/migrations/0001_initial.py index 3881db0c7d6..972b5191224 100644 --- a/geonode/geoapps/migrations/0001_initial.py +++ b/geonode/geoapps/migrations/0001_initial.py @@ -2,6 +2,7 @@ from django.db import migrations, models import django.db.models.deletion +from django.db.models.fields.json import JSONField class Migration(migrations.Migration): @@ -35,7 +36,7 @@ class Migration(migrations.Migration): name='GeoAppData', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('blob', models.JSONField(default={})), + ('blob', JSONField(default=dict)), ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='geoapps.GeoApp')), ], ), diff --git a/geonode/geoapps/migrations/0002_auto_20210615_0717.py b/geonode/geoapps/migrations/0002_auto_20210615_0717.py new file mode 100644 index 00000000000..5371b8fb673 --- /dev/null +++ b/geonode/geoapps/migrations/0002_auto_20210615_0717.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2 on 2021-06-15 07:17 + +from django.db import migrations + + +move_data_blob = ''' +UPDATE base_resourcebase +SET "blob"=subquery."blob" +FROM (select resource_id,"blob" from geoapps_geoappdata gg) AS subquery +WHERE base_resourcebase.id=subquery.resource_id; +''' + +class Migration(migrations.Migration): + + dependencies = [ + ('geoapps', '0001_initial'), + ('base', '0066_resourcebase_data'), + ] + + operations = [ + migrations.RunSQL(move_data_blob), + migrations.RemoveField( + model_name='geoapp', + name='data', + ), + migrations.DeleteModel( + name='GeoAppData', + ), + ] diff --git a/geonode/geoapps/migrations/0003_alter_geoapp_name.py b/geonode/geoapps/migrations/0003_alter_geoapp_name.py new file mode 100644 index 00000000000..8d13b7d39fc --- /dev/null +++ b/geonode/geoapps/migrations/0003_alter_geoapp_name.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-06-22 14:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('geoapps', '0002_auto_20210615_0717'), + ] + + operations = [ + migrations.AlterField( + model_name='geoapp', + name='name', + field=models.TextField(verbose_name='Name'), + ), + ] diff --git a/geonode/geoapps/migrations/0003_update_detail_url.py b/geonode/geoapps/migrations/0003_update_detail_url.py new file mode 100644 index 00000000000..631517785b1 --- /dev/null +++ b/geonode/geoapps/migrations/0003_update_detail_url.py @@ -0,0 +1,21 @@ +from django.db import migrations +from django.db.models import Value +from django.db.models.functions import Concat +from django.db.migrations.operations import RunPython + +from geonode.geoapps.models import GeoApp + + +def update_geoapps_detail_url(apps, schema_editor): + GeoApp.objects.update(detail_url=Concat(Value('/apps/'), 'id')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('geoapps', '0002_auto_20210615_0717'), + ] + + operations = [ + RunPython(update_geoapps_detail_url) + ] diff --git a/geonode/geoapps/migrations/0004_merge_0003_alter_geoapp_name_0003_update_detail_url.py b/geonode/geoapps/migrations/0004_merge_0003_alter_geoapp_name_0003_update_detail_url.py new file mode 100644 index 00000000000..15307160446 --- /dev/null +++ b/geonode/geoapps/migrations/0004_merge_0003_alter_geoapp_name_0003_update_detail_url.py @@ -0,0 +1,14 @@ +# Generated by Django 3.2.4 on 2021-06-23 08:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('geoapps', '0003_alter_geoapp_name'), + ('geoapps', '0003_update_detail_url'), + ] + + operations = [ + ] diff --git a/geonode/geoapps/migrations/__init__.py b/geonode/geoapps/migrations/__init__.py index 725b3926923..47996945d44 100644 --- a/geonode/geoapps/migrations/__init__.py +++ b/geonode/geoapps/migrations/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo diff --git a/geonode/geoapps/models.py b/geonode/geoapps/models.py index ef956d21880..a1ffc3cc630 100644 --- a/geonode/geoapps/models.py +++ b/geonode/geoapps/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ######################################################################### # # Copyright (C) 2020 OSGeo @@ -21,12 +20,11 @@ from django.db import models from django.urls import reverse -from django.db.models import signals from django.utils.translation import ugettext_lazy as _ from guardian.shortcuts import get_anonymous_user -from geonode.base.models import ResourceBase, resourcebase_post_save +from geonode.base.models import ResourceBase logger = logging.getLogger("geonode.geoapps.models") @@ -45,7 +43,7 @@ class GeoApp(ResourceBase): ] } - name = models.TextField(_('Name'), unique=True, db_index=True) + name = models.TextField(_('Name'), null=False, blank=False) # viewer configuration zoom = models.IntegerField(_('zoom'), null=True, blank=True) @@ -71,13 +69,6 @@ class GeoApp(ResourceBase): # Alphanumeric alternative to referencing geoapps by id, appended to end of # URL instead of id, ie http://domain/geoapps/someview - data = models.OneToOneField( - "GeoAppData", - related_name="data", - null=True, - blank=True, - on_delete=models.CASCADE) - def __str__(self): return f'{self.title} by {(self.owner.username if self.owner else "")}' @@ -129,23 +120,3 @@ def get_absolute_url(self): @property def embed_url(self): return reverse('geoapp_embed', kwargs={'geoappid': self.pk}) - - class Meta(ResourceBase.Meta): - pass - - -class GeoAppData(models.Model): - - blob = models.JSONField( - null=False, - default={}) - - resource = models.ForeignKey( - GeoApp, - null=False, - blank=False, - on_delete=models.CASCADE) - - -# signals.pre_delete.connect(pre_delete_app, sender=GeoApp) -signals.post_save.connect(resourcebase_post_save, sender=GeoApp) diff --git a/geonode/geoapps/templates/apps/app_detail.html b/geonode/geoapps/templates/apps/app_detail.html index 60873135a56..1078da05381 100644 --- a/geonode/geoapps/templates/apps/app_detail.html +++ b/geonode/geoapps/templates/apps/app_detail.html @@ -95,9 +95,9 @@

{% trans 'Average Rating' %}

{% if not READ_ONLY_MODE %} {% if "change_resourcebase" in perms_list or "change_resourcebase_metadata" in perms_list %}
  • - +
    • -