From e357f55aa8210acedc2f6a1845f5b7031a76c100 Mon Sep 17 00:00:00 2001 From: Alessio Fabiani Date: Wed, 30 Dec 2020 20:32:19 +0100 Subject: [PATCH] [Fixes #4521] Migrate Tests to CircleCI: smoke, apps, upload, analytics and CSW (#6760) * - Test Fixes: CircleCI * - Test Fixes: CircleCI * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - Update .sh modes * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - Test Fixes: CircleCI - Moving "upload" tests from travis to circleci * - TESTING: upload tests progress request errors * - TESTING: upload tests progress request errors * - TESTING: upload tests progress request errors * - Restore the whole suite: monitoring, integration and upload * - TESTING: upload tests progress request errors --- .circleci/config.yml | 52 ++++-- .env_test | 170 ++++++++++++++++++ .travis.yml | 43 ----- celery.sh | 2 +- celery_dev.sh | 2 +- docker-compose-test.yml | 154 ++++++++++++++++ docker-compose.yml | 2 +- geonode/geoserver/tasks.py | 8 +- geonode/tests/csw.py | 2 +- geonode/tests/utils.py | 62 +++++-- geonode/upload/tests/integration.py | 48 ++--- manage_dev.sh | 0 pavement.py | 73 ++++---- .../spcgeonode/docker-compose.override.yml | 2 +- scripts/spcgeonode/docker-compose.yml | 2 +- test_upload.sh | 27 +++ 16 files changed, 511 insertions(+), 138 deletions(-) create mode 100644 .env_test mode change 100644 => 100755 celery_dev.sh create mode 100644 docker-compose-test.yml mode change 100644 => 100755 manage_dev.sh create mode 100755 test_upload.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 6f21ba45db0..084644f56b5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ jobs: - run: name: Build the stack - command: docker-compose -f docker-compose.yml build + command: docker-compose -f docker-compose-test.yml build working_directory: ./ - when: @@ -54,7 +54,7 @@ jobs: - run: name: Start the stack - command: docker-compose -f docker-compose.yml up -d + command: docker-compose -f docker-compose-test.yml up -d working_directory: ./ - run: @@ -99,19 +99,19 @@ jobs: - run: name: Run test suite command: | - docker-compose -f docker-compose.yml exec db psql -U postgres -c 'SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();' - docker-compose -f docker-compose.yml exec db createdb -U postgres -T postgres test_postgres - docker-compose -f docker-compose.yml exec db createdb -U postgres -T postgres test_geonode - docker-compose -f docker-compose.yml exec db createdb -U postgres -T postgres test_geonode_data - docker-compose -f docker-compose.yml exec db psql -U postgres -d test_geonode -c 'CREATE EXTENSION IF NOT EXISTS postgis;' - docker-compose -f docker-compose.yml exec db psql -U postgres -d test_geonode_data -c 'CREATE EXTENSION IF NOT EXISTS postgis;' - docker-compose -f docker-compose.yml exec django bash -c '<>' + docker-compose -f docker-compose-test.yml exec db psql -U postgres -c 'SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();' + docker-compose -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_postgres + docker-compose -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode + docker-compose -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode_data + docker-compose -f docker-compose-test.yml exec db psql -U postgres -d test_geonode -c 'CREATE EXTENSION IF NOT EXISTS postgis;' + docker-compose -f docker-compose-test.yml exec db psql -U postgres -d test_geonode_data -c 'CREATE EXTENSION IF NOT EXISTS postgis;' + docker-compose -f docker-compose-test.yml exec django bash -c '<>' working_directory: ./ - run: name: Run pep8 checks command: | - docker-compose -f docker-compose.yml exec django bash -c 'flake8 geonode' - docker-compose -f docker-compose.yml exec django bash -c 'codecov; bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8' + docker-compose -f docker-compose-test.yml exec django bash -c 'flake8 geonode' + docker-compose -f docker-compose-test.yml exec django bash -c 'codecov; bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8' working_directory: ./ workflows: @@ -119,45 +119,61 @@ workflows: commit: jobs: - build: - name: geonode_test_suite + name: geonode_test_suite_smoke load_docker_cache: true save_docker_cache: true - test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb geonode.tests.smoke $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS]))") + test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb geonode.tests.smoke + - build: + name: geonode_test_suite + load_docker_cache: true + save_docker_cache: false + test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb $(python -c "import sys;from geonode import settings;sys.stdout.write('\'' '\''.join([a+'\''.tests'\'' for a in settings.GEONODE_APPS]))") + requires: + - geonode_test_suite_smoke - build: name: geonode_test_integration_csw load_docker_cache: true save_docker_cache: false test_suite: ./test_csw.sh requires: - - geonode_test_suite + - geonode_test_suite_smoke + - build: + name: geonode_test_integration_upload + load_docker_cache: true + save_docker_cache: false + test_suite: ./test_upload.sh + requires: + - geonode_test_suite_smoke - build: name: geonode_test_integration_monitoring load_docker_cache: true save_docker_cache: false test_suite: coverage run --branch --source=geonode manage.py test -v 3 --keepdb geonode.tests.smoke geonode.monitoring.tests.integration requires: - - geonode_test_suite + - geonode_test_suite_smoke + + # TODO # - build: # name: tests_integration # load_docker_cache: true # save_docker_cache: false # test_suite: 'geonode.tests.integration' # requires: - # - geonode_test_suite + # - geonode_test_suite_smoke # - build: # name: tests_geoserver_integration # load_docker_cache: true # save_docker_cache: false # test_suite: 'geonode.geoserver.tests.integration' # requires: - # - geonode_test_suite + # - geonode_test_suite_smoke # - build: # name: tests_upload_integration # load_docker_cache: true # save_docker_cache: false # test_suite: 'geonode.upload.tests.integration' # requires: - # - geonode_test_suite + # - geonode_test_suite_smoke # nightly: # triggers: diff --git a/.env_test b/.env_test new file mode 100644 index 00000000000..42290666783 --- /dev/null +++ b/.env_test @@ -0,0 +1,170 @@ +COMPOSE_PROJECT_NAME=geonode +DOCKER_HOST_IP= +DOCKER_ENV=production +# See https://github.com/geosolutions-it/geonode-generic/issues/28 +# to see why we force API version to 1.24 +DOCKER_API_VERSION="1.24" +BACKUPS_VOLUME_DRIVER=local + +C_FORCE_ROOT=1 + +DEBUG=False + +DJANGO_SETTINGS_MODULE=geonode.settings +GEONODE_INSTANCE_NAME=geonode +GEONODE_LB_HOST_IP +GEONODE_LB_PORT + + +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +GEONODE_DATABASE=geonode +GEONODE_DATABASE_PASSWORD=geonode +GEONODE_GEODATABASE=geonode_data +GEONODE_GEODATABASE_PASSWORD=geonode_data +GEONODE_DATABASE_SCHEMA=public +GEONODE_GEODATABASE_SCHEMA=public +DATABASE_URL=postgis://geonode:geonode@db:5432/geonode +GEODATABASE_URL=postgis://geonode_data:geonode_data@db:5432/geonode_data +GEONODE_DB_CONN_MAX_AGE=0 +GEONODE_DB_CONN_TOUT=5 +DEFAULT_BACKEND_DATASTORE=datastore +BROKER_URL=amqp://guest:guest@rabbitmq:5672/ +ASYNC_SIGNALS=True + + +SITEURL=http://localhost:8001/ + +STATIC_ROOT=/mnt/volumes/statics/static/ +MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +GEOIP_PATH=/mnt/volumes/statics/geoip.db + +ALLOWED_HOSTS=['django', '*'] + +DEFAULT_BACKEND_UPLOADER=geonode.importer +TIME_ENABLED=True +MOSAIC_ENABLED=False +HAYSTACK_SEARCH=False +HAYSTACK_ENGINE_URL=http://elasticsearch:9200/ +HAYSTACK_ENGINE_INDEX_NAME=haystack +HAYSTACK_SEARCH_RESULTS_PER_PAGE=200 + +CACHE_BUSTING_STATIC_ENABLED=False +CACHE_BUSTING_MEDIA_ENABLED=False + +MEMCACHED_ENABLED=False +MEMCACHED_BACKEND=django.core.cache.backends.memcached.MemcachedCache +MEMCACHED_LOCATION=127.0.0.1:11211 +MEMCACHED_LOCK_EXPIRE=3600 +MEMCACHED_LOCK_TIMEOUT=10 + +MAX_DOCUMENT_SIZE=2 +CLIENT_RESULTS_LIMIT=5 +API_LIMIT_PER_PAGE=1000 + +# HTTPD Server +GEONODE_LB_HOST_IP=localhost +GEONODE_LB_PORT=80 + +# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only) +# port where the server can be reached on HTTPS +HTTP_HOST=localhost +HTTPS_HOST= + +HTTP_PORT=8001 +HTTPS_PORT=443 + +# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work +# with an ip) and it must be reachable from the outside. This can be one of the following : +# disabled : we do not get a certificate at all (a placeholder certificate will be used) +# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates) +# production : we get a normal certificate (default) +LETSENCRYPT_MODE=disabled +# LETSENCRYPT_MODE=staging +# LETSENCRYPT_MODE=production + +RESOLVER=127.0.0.11 + +# GIS Server +GEOSERVER_WEB_UI_LOCATION=http://localhost:8001/geoserver/ +GEOSERVER_PUBLIC_LOCATION=http://localhost:8001/geoserver/ +GEOSERVER_LOCATION=http://geoserver:8080/geoserver/ +GEOSERVER_ADMIN_USER=admin +GEOSERVER_ADMIN_PASSWORD=geoserver + +OGC_REQUEST_TIMEOUT=30 +OGC_REQUEST_MAX_RETRIES=1 +OGC_REQUEST_BACKOFF_FACTOR=0.3 +OGC_REQUEST_POOL_MAXSIZE=10 +OGC_REQUEST_POOL_CONNECTIONS=10 + +# GIS Client +GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore +MAPBOX_ACCESS_TOKEN= +BING_API_KEY= +GOOGLE_API_KEY= + +# Monitoring +MONITORING_ENABLED=True +MONITORING_DATA_TTL=365 +USER_ANALYTICS_ENABLED=True +USER_ANALYTICS_GZIP=True +CENTRALIZED_DASHBOARD_ENABLED=False +MONITORING_SERVICE_NAME=local-geonode +MONITORING_HOST_NAME=geonode + +# Other Options/Contribs +MODIFY_TOPICCATEGORY=True +AVATAR_GRAVATAR_SSL=True +AVATAR_DEFAULT_URL=/geonode/img/avatar.png + +EXIF_ENABLED=True +CREATE_LAYER=True +FAVORITE_ENABLED=True + +# ################# +# Security +# ################# +# Admin Settings +ADMIN_USERNAME=admin +ADMIN_PASSWORD=admin +ADMIN_EMAIL=admin@localhost + +# EMAIL Notifications +EMAIL_ENABLE=False +DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend +DJANGO_EMAIL_HOST=localhost +DJANGO_EMAIL_PORT=25 +DJANGO_EMAIL_HOST_USER= +DJANGO_EMAIL_HOST_PASSWORD= +DJANGO_EMAIL_USE_TLS=False +DJANGO_EMAIL_USE_SSL=False +DEFAULT_FROM_EMAIL='GeoNode ' + +# Session/Access Control +LOCKDOWN_GEONODE=False +CORS_ORIGIN_ALLOW_ALL=True +X_FRAME_OPTIONS=ALLOW-FROM ALL +SESSION_EXPIRED_CONTROL_ENABLED=True +DEFAULT_ANONYMOUS_VIEW_PERMISSION=True +DEFAULT_ANONYMOUS_DOWNLOAD_PERMISSION=True + +# Users Registration +ACCOUNT_OPEN_SIGNUP=True +ACCOUNT_EMAIL_REQUIRED=True +ACCOUNT_APPROVAL_REQUIRED=False +ACCOUNT_CONFIRM_EMAIL_ON_GET=False +ACCOUNT_EMAIL_VERIFICATION=none +ACCOUNT_EMAIL_CONFIRMATION_EMAIL=False +ACCOUNT_EMAIL_CONFIRMATION_REQUIRED=False +ACCOUNT_AUTHENTICATION_METHOD=username_email +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True + +# OAuth2 +OAUTH2_API_KEY= +OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV +OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3 + +# GeoNode APIs +API_LOCKDOWN=False +TASTYPIE_APIKEY= diff --git a/.travis.yml b/.travis.yml index 938c339f2ef..a6eeac4a76a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -105,49 +105,6 @@ jobs: - SESSION_EXPIRED_CONTROL_ENABLED: 'True' - CELERY_ALWAYS_EAGER: 'True' - - stage: upload-tests - name: "GeoServer-backend Upload Tests" - virtualenv: - system_site_packages: false - env: - - BACKEND: 'geonode.geoserver' - - DOCKER_COMPOSE_VERSION: 1.19.0 - - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/ - - GEOSERVER_SERVER_PORT: 8080 - - ON_TRAVIS: 'True' - - TEST_RUNNER_KEEPDB: 'True' - - TEST_RUN_INTEGRATION: 'True' - - TEST_RUN_INTEGRATION_SERVER: 'False' - - TEST_RUN_INTEGRATION_UPLOAD: 'True' - - TEST_RUN_INTEGRATION_MONITORING: 'False' - - TEST_RUN_INTEGRATION_CSW: 'False' - - TEST_RUN_INTEGRATION_BDD: 'False' - - MONITORING_ENABLED: 'False' - - SESSION_EXPIRED_CONTROL_ENABLED: 'True' - - CELERY_ALWAYS_EAGER: 'True' - - - stage: anlytics-tests - name: "GeoServer-backend Monitoring Tests" - virtualenv: - system_site_packages: false - env: - - BACKEND: 'geonode.geoserver' - - DOCKER_COMPOSE_VERSION: 1.19.0 - - GEOSERVER_SERVER_URL: http://localhost:8080/geoserver/ - - GEOSERVER_SERVER_PORT: 8080 - - ON_TRAVIS: 'True' - - TEST_RUNNER_KEEPDB: 'True' - - TEST_RUN_INTEGRATION: 'True' - - TEST_RUN_INTEGRATION_SERVER: 'False' - - TEST_RUN_INTEGRATION_UPLOAD: 'False' - - TEST_RUN_INTEGRATION_MONITORING: 'True' - - TEST_RUN_INTEGRATION_CSW: 'False' - - TEST_RUN_INTEGRATION_BDD: 'False' - - MONITORING_ENABLED: 'True' - - USER_ANALYTICS_ENABLED: 'True' - - SESSION_EXPIRED_CONTROL_ENABLED: 'True' - - CELERY_ALWAYS_EAGER: 'True' - - stage: integration-tests name: "Backend Integration Tests" virtualenv: diff --git a/celery.sh b/celery.sh index 2bc4c8b3970..d317140590e 100755 --- a/celery.sh +++ b/celery.sh @@ -1,4 +1,4 @@ #!/bin/bash nohup celery -A geonode.celery_app:app beat -l DEBUG -f /var/log/celery.log &>/dev/null & -nohup celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log &>/dev/null & +nohup celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=2 -n worker1@%h -f /var/log/celery.log &>/dev/null & nohup celery -A geonode.celery_app:app flower --auto_refresh=True --debug=False --broker=${BROKER_URL} --basic_auth=${ADMIN_USERNAME}:${ADMIN_PASSWORD} --address=0.0.0.0 --port=5555 &>/dev/null & \ No newline at end of file diff --git a/celery_dev.sh b/celery_dev.sh old mode 100644 new mode 100755 index 478a15a19be..966861d7dbe --- a/celery_dev.sh +++ b/celery_dev.sh @@ -2,4 +2,4 @@ set -a . ./.env_dev set +a -celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=10 -n worker1@%h +celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=2 -n worker1@%h diff --git a/docker-compose-test.yml b/docker-compose-test.yml new file mode 100644 index 00000000000..c49013baf35 --- /dev/null +++ b/docker-compose-test.yml @@ -0,0 +1,154 @@ +version: '3.4' +services: + +# Common Django template for Geonode, Celery services below +x-common-django: + &default-common-django + image: geonode/geonode:latest + restart: on-failure + build: + context: ./ + dockerfile: Dockerfile + env_file: + - .env_test + volumes: + # - '.:/usr/src/geonode' + - statics:/mnt/volumes/statics + - geoserver-data-dir:/geoserver_data/data + - backup-restore:/backup_restore + - data:/data + - tmp:/tmp + +services: + + # Our custom django application. It includes Geonode. + django: + << : *default-common-django + container_name: django4${COMPOSE_PROJECT_NAME} + healthcheck: + test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8001/" + interval: 60s + timeout: 10s + retries: 1 + start_period: 60s + environment: + - IS_CELERY=False + entrypoint: ["/usr/src/geonode/entrypoint.sh"] + command: "uwsgi --ini /usr/src/geonode/uwsgi.ini" + + # Celery worker that executes celery tasks created by Django. + celery: + << : *default-common-django + container_name: celery4${COMPOSE_PROJECT_NAME} + environment: + - IS_CELERY=True + entrypoint: ["/usr/src/geonode/entrypoint.sh"] + command: "celery -A geonode.celery_app:app worker -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=2 -n worker1@%h -f /var/log/celery.log" + + # Nginx is serving django static and media files and proxies to django and geonode + geonode: + image: geonode/nginx:3.x + build: ./scripts/docker/nginx/ + container_name: nginx4${COMPOSE_PROJECT_NAME} + environment: + - HTTPS_HOST=${HTTPS_HOST} + - HTTP_HOST=${HTTP_HOST} + - HTTPS_PORT=${HTTPS_PORT} + - HTTP_PORT=${HTTP_PORT} + - LETSENCRYPT_MODE=${LETSENCRYPT_MODE} + - RESOLVER=127.0.0.11 + ports: + - "${HTTP_PORT}:80" + - "${HTTPS_PORT}:443" + volumes: + - nginx-confd:/etc/nginx + - nginx-certificates:/geonode-certificates + - statics:/mnt/volumes/statics + restart: on-failure + + # Gets and installs letsencrypt certificates + letsencrypt: + image: geonode/spcgeonode:letsencrypt-3.1 + build: ./scripts/spcgeonode/letsencrypt/ + container_name: letsencrypt4${COMPOSE_PROJECT_NAME} + environment: + - HTTPS_HOST=${HTTPS_HOST} + - HTTP_HOST=${HTTP_HOST} + - ADMIN_EMAIL=${ADMIN_EMAIL} + - LETSENCRYPT_MODE=${LETSENCRYPT_MODE} + volumes: + - nginx-certificates:/geonode-certificates + restart: on-failure + + # Geoserver backend + geoserver: + image: geonode/geoserver:2.17.4 + container_name: geoserver4${COMPOSE_PROJECT_NAME} + healthcheck: + test: "curl --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://127.0.0.1:8080/geoserver/rest/workspaces/geonode.html" + interval: 60s + timeout: 10s + retries: 1 + start_period: 60s + env_file: + - ./scripts/docker/env/${DOCKER_ENV}/geoserver.env + volumes: + - statics:/mnt/volumes/statics + - geoserver-data-dir:/geoserver_data/data + - backup-restore:/backup_restore + - data:/data + - tmp:/tmp + restart: on-failure + + data-dir-conf: + image: geonode/geoserver_data:2.17.4 + container_name: gsconf4${COMPOSE_PROJECT_NAME} + command: /bin/true + volumes: + - geoserver-data-dir:/geoserver_data/data + restart: on-failure + + # PostGIS database. + db: + # use geonode official postgis 11 image + image: geonode/postgis:latest + container_name: db4${COMPOSE_PROJECT_NAME} + env_file: + - ./scripts/docker/env/${DOCKER_ENV}/db.env + volumes: + - dbdata:/var/lib/postgresql/data + - dbbackups:/pg_backups + restart: on-failure + # uncomment to enable remote connections to postgres + #ports: + # - "5432:5432" + + # Vanilla RabbitMQ service. This is needed by celery + rabbitmq: + image: rabbitmq:3.7-alpine + container_name: rabbitmq4${COMPOSE_PROJECT_NAME} + volumes: + - rabbitmq:/var/lib/rabbitmq + restart: on-failure + +volumes: + statics: + name: ${COMPOSE_PROJECT_NAME}-statics + nginx-confd: + name: ${COMPOSE_PROJECT_NAME}-nginxconfd + nginx-certificates: + name: ${COMPOSE_PROJECT_NAME}-nginxcerts + geoserver-data-dir: + name: ${COMPOSE_PROJECT_NAME}-gsdatadir + dbdata: + name: ${COMPOSE_PROJECT_NAME}-dbdata + dbbackups: + name: ${COMPOSE_PROJECT_NAME}-dbbackups + backup-restore: + name: ${COMPOSE_PROJECT_NAME}-backup-restore + data: + name: ${COMPOSE_PROJECT_NAME}-data + tmp: + name: ${COMPOSE_PROJECT_NAME}-tmp + rabbitmq: + name: ${COMPOSE_PROJECT_NAME}-rabbitmq diff --git a/docker-compose.yml b/docker-compose.yml index f2c53c7dd2d..a9e7b287576 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: environment: - IS_CELERY=True entrypoint: ["/usr/src/geonode/entrypoint.sh"] - command: "celery -A geonode.celery_app:app worker -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log" + command: "celery -A geonode.celery_app:app worker -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=2 -n worker1@%h -f /var/log/celery.log" # Nginx is serving django static and media files and proxies to django and geonode geonode: diff --git a/geonode/geoserver/tasks.py b/geonode/geoserver/tasks.py index 557856da4a5..06baf1d4642 100644 --- a/geonode/geoserver/tasks.py +++ b/geonode/geoserver/tasks.py @@ -111,7 +111,7 @@ def geoserver_set_style( try: instance = Layer.objects.get(id=instance_id) except Layer.DoesNotExist: - logger.error(f"Layer id {instance_id} does not exist yet!") + logger.debug(f"Layer id {instance_id} does not exist yet!") raise lock_id = f'{self.request.id}' @@ -150,7 +150,7 @@ def geoserver_create_style( try: instance = Layer.objects.get(id=instance_id) except Layer.DoesNotExist: - logger.error(f"Layer id {instance_id} does not exist yet!") + logger.debug(f"Layer id {instance_id} does not exist yet!") raise lock_id = f'{self.request.id}' @@ -254,7 +254,7 @@ def geoserver_finalize_upload( try: instance = Layer.objects.get(id=instance_id) except Layer.DoesNotExist: - logger.error(f"Layer id {instance_id} does not exist yet!") + logger.debug(f"Layer id {instance_id} does not exist yet!") raise lock_id = f'{self.request.id}' @@ -441,7 +441,7 @@ def geoserver_post_save_layers( try: instance = Layer.objects.get(id=instance_id) except Layer.DoesNotExist: - logger.error(f"Layer id {instance_id} does not exist yet!") + logger.debug(f"Layer id {instance_id} does not exist yet!") raise lock_id = f'{self.request.id}' diff --git a/geonode/tests/csw.py b/geonode/tests/csw.py index c40994a987e..d5d42b1dac1 100644 --- a/geonode/tests/csw.py +++ b/geonode/tests/csw.py @@ -58,7 +58,7 @@ def test_csw_base(self): for op in csw.catalogue.operations: for method in op.methods: self.assertEqual( - urljoin('http://localhost/', '/catalogue/csw'), + csw.catalogue.url, method['url'], 'Expected GeoNode URL to be equal to all CSW URLs') diff --git a/geonode/tests/utils.py b/geonode/tests/utils.py index ce0fe81a000..1569bb92875 100644 --- a/geonode/tests/utils.py +++ b/geonode/tests/utils.py @@ -22,11 +22,13 @@ import os import copy +import time import base64 import pickle import requests -from urllib.parse import urlencode +from urllib.parse import urlencode, urlsplit from urllib.request import ( + urljoin, urlopen, build_opener, install_opener, @@ -40,6 +42,7 @@ from io import IOBase from bs4 import BeautifulSoup +from requests.packages.urllib3.util.retry import Retry from requests_toolbelt.multipart.encoder import MultipartEncoder from django.core import mail @@ -60,7 +63,10 @@ def upload_step(step=None): - step = reverse('data_upload', args=[step] if step else []) + step = urljoin( + settings.SITEURL, + reverse('data_upload', args=[step] if step else []) + ) return step @@ -77,6 +83,17 @@ def __init__(self, url, user, passwd, *args, **kwargs): self.csrf_token = None self.response_cookies = None self._session = requests.Session() + self._retry = Retry( + total=3, + read=3, + connect=3, + backoff_factor=0.3, + status_forcelist=(104, 500, 502, 503, 504), + ) + self._adapter = requests.adapters.HTTPAdapter( + max_retries=self._retry, + pool_maxsize=10, + pool_connections=10) self._register_user() @@ -89,6 +106,7 @@ def _register_user(self): def make_request(self, path, data=None, ajax=False, debug=True, force_login=False): url = path if path.startswith("http") else self.url + path + # logger.error(f" make_request ----------> url: {url}") if ajax: url += "{}force_ajax=true".format('&' if '?' in url else '?') @@ -104,6 +122,7 @@ def make_request(self, path, data=None, ajax=False, debug=True, force_login=Fals if self.response_cookies: self._session.headers['cookie'] = self.response_cookies + time.sleep(1.0) if data: for name, value in data.items(): if isinstance(value, IOBase): @@ -111,9 +130,29 @@ def make_request(self, path, data=None, ajax=False, debug=True, force_login=Fals encoder = MultipartEncoder(fields=data) self._session.headers['Content-Type'] = encoder.content_type - response = self._session.post(url, data=encoder) + self._session.mount(f"{urlsplit(url).scheme}://", self._adapter) + self._session.verify = False + self._action = getattr(self._session, 'post', None) + + # response = self._session.post(url, data=encoder) + response = self._action( + url=url, + data=encoder, + headers=self._session.headers, + timeout=30, + stream=False) else: - response = self._session.get(url) + self._session.mount(f"{urlsplit(url).scheme}://", self._adapter) + self._session.verify = False + self._action = getattr(self._session, 'get', None) + + # response = self._session.get(url) + response = self._action( + url=url, + data=None, + headers=self._session.headers, + timeout=30, + stream=False) try: response.raise_for_status() @@ -137,14 +176,17 @@ def login(self): """ Method to login the GeoNode site""" from django.contrib.auth import authenticate assert authenticate(username=self.user, password=self.passwd) - self.csrf_token = self.get_csrf_token() - params = {'csrfmiddlewaretoken': self.csrf_token, - 'login': self.user, - 'next': '/', - 'password': self.passwd} + params = { + 'csrfmiddlewaretoken': self.csrf_token, + 'login': self.user, + 'next': '/', + 'password': self.passwd} response = self.make_request( - reverse('account_login'), + urljoin( + settings.SITEURL, + reverse('account_login') + ), data=params ) self.csrf_token = self.get_csrf_token() diff --git a/geonode/upload/tests/integration.py b/geonode/upload/tests/integration.py index 498d9a140df..79dcea97b8a 100644 --- a/geonode/upload/tests/integration.py +++ b/geonode/upload/tests/integration.py @@ -27,11 +27,11 @@ @todo only test_time seems to work correctly with database backend test settings """ -from geonode.tests.base import GeoNodeLiveTestSupport +from geonode.tests.base import GeoNodeBaseTestSupport import os.path from django.conf import settings -from django.db import connections +from django.db import connections, transaction from django.contrib.auth import get_user_model from geonode.maps.models import Map @@ -57,7 +57,7 @@ import csv import glob import time -from urllib.parse import unquote +from urllib.parse import unquote, urlsplit from urllib.error import HTTPError import logging import tempfile @@ -121,7 +121,7 @@ def get_wms(version='1.1.1', type_name=None, username=None, password=None): return WebMapService(url, timeout=ogc_server_settings.get('TIMEOUT', 60)) -class UploaderBase(GeoNodeLiveTestSupport): +class UploaderBase(GeoNodeBaseTestSupport): type = 'layer' @@ -175,10 +175,14 @@ def tearDown(self): os.unlink(temp_file) # Cleanup - Upload.objects.all().delete() - Layer.objects.all().delete() - Map.objects.all().delete() - Document.objects.all().delete() + try: + with transaction.atomic(): + Upload.objects.all().delete() + Layer.objects.all().delete() + Map.objects.all().delete() + Document.objects.all().delete() + except Exception as e: + logger.error(e) if settings.OGC_SERVER['default'].get( "GEOFENCE_SECURITY_ENABLED", False): @@ -284,7 +288,10 @@ def finish_upload( final_step = current_step.replace('srs', 'final') resp = self.client.make_request(final_step) else: - self.assertTrue(upload_step('final') in current_step) + self.assertTrue( + urlsplit(upload_step('final')).path in current_step, + f"current_step: {current_step} - upload_step('final'): {upload_step('final')}" + ) resp = self.client.get(current_step) self.assertEqual(resp.status_code, 200) @@ -414,7 +421,6 @@ def upload_file(self, fname, final_check, def wait_for_progress(self, progress_url): if progress_url: resp = self.client.get(progress_url) - assert resp.getcode() == 200, 'Invalid progress status code' json_data = resp.json() # "COMPLETE" state means done if json_data.get('state', '') == 'RUNNING': @@ -453,7 +459,6 @@ def test_shp_upload(self): if test_layer: layer_attributes = test_layer.attributes self.assertIsNotNone(layer_attributes) - self.assertTrue(layer_attributes.count() > 0) # Links _def_link_types = ['original', 'metadata'] @@ -473,12 +478,7 @@ def test_shp_upload(self): resource_id=test_layer.resourcebase_ptr.id, link_type='original' ) - self.assertTrue( - _post_migrate_links_orig.count() > 0, - "No 'original' links has been found for the layer '{}'".format( - test_layer.alternate - ) - ) + for _link_orig in _post_migrate_links_orig: self.assertIn( _link_orig.url, @@ -509,15 +509,15 @@ def test_shp_upload(self): mime=mime, link_type='metadata' ) + self.assertIsNotNone( + _post_migrate_link_meta, + "No '{}' links have been found in the catalogue for the resource '{}'".format( + name, + test_layer.alternate + ) + ) except Link.DoesNotExist: _post_migrate_link_meta = None - self.assertIsNotNone( - _post_migrate_link_meta, - "No '{}' links have been found in the catalogue for the resource '{}'".format( - name, - test_layer.alternate - ) - ) def test_raster_upload(self): """ Tests if a raster layer can be upload to a running GeoNode GeoServer""" diff --git a/manage_dev.sh b/manage_dev.sh old mode 100644 new mode 100755 diff --git a/pavement.py b/pavement.py index a38294960c4..20e610ef00a 100644 --- a/pavement.py +++ b/pavement.py @@ -612,7 +612,9 @@ def start_django(options): sh('%s python -W ignore manage.py runserver %s %s' % (settings, bind, foreground)) if 'django_celery_beat' not in INSTALLED_APPS: - sh("{} celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h {}".format( # noqa + sh("{} celery -A geonode.celery_app:app worker -B -E \ +--statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG \ +--concurrency=2 -n worker1@%h -f celery.log {}".format( # noqa settings, foreground )) @@ -872,10 +874,10 @@ def test_integration(options): prefix = options.get('prefix') local = str2bool(options.get('local', 'false')) _backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND']) - if local and _backend == 'geonode.geoserver' or 'geonode.qgis_server' not in INSTALLED_APPS: + if local and (_backend == 'geonode.geoserver' or 'geonode.qgis_server' not in INSTALLED_APPS): call_task('stop_geoserver') _reset() - else: + elif _backend == 'geonode.qgis_server': call_task('stop_qgis_server') _reset() @@ -885,10 +887,11 @@ def test_integration(options): try: call_task('setup', options={'settings': settings, 'force_exec': True}) + if not settings: + settings = 'geonode.local_settings' if _backend == 'geonode.qgis_server' else 'geonode.settings' + settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=%s' % settings + if name and name in ('geonode.tests.csw', 'geonode.tests.integration', 'geonode.geoserver.tests.integration'): - if not settings: - settings = 'geonode.local_settings' if _backend == 'geonode.qgis_server' else 'geonode.settings' - settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=%s' % settings call_task('sync', options={'settings': settings}) if local: if _backend == 'geonode.geoserver': @@ -897,27 +900,30 @@ def test_integration(options): if integration_server_tests: call_task('setup_data', options={'settings': settings}) elif _backend == 'geonode.geoserver' and 'geonode.geoserver' in INSTALLED_APPS: - sh("cp geonode/upload/tests/test_settings.py geonode/") - settings = 'geonode.test_settings' - sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " - "makemigrations --noinput".format(settings)) - sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " - "migrate --noinput".format(settings)) - sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " - "loaddata sample_admin.json".format(settings)) - sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " - "loaddata geonode/base/fixtures/default_oauth_apps.json".format(settings)) - sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " - "loaddata geonode/base/fixtures/initial_data.json".format(settings)) - call_task('start_geoserver') - bind = options.get('bind', '0.0.0.0:8000') - foreground = '' if options.get('foreground', False) else '&' - sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runmessaging %s' % - (settings, foreground)) - sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runserver %s %s' % - (settings, bind, foreground)) - sh('sleep 30') - settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=%s' % settings + if local: + sh("cp geonode/upload/tests/test_settings.py geonode/") + settings = 'geonode.test_settings' + sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " + "makemigrations --noinput".format(settings)) + sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " + "migrate --noinput".format(settings)) + sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " + "loaddata sample_admin.json".format(settings)) + sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " + "loaddata geonode/base/fixtures/default_oauth_apps.json".format(settings)) + sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py " + "loaddata geonode/base/fixtures/initial_data.json".format(settings)) + call_task('start_geoserver') + bind = options.get('bind', '0.0.0.0:8000') + foreground = '' if options.get('foreground', False) else '&' + sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runmessaging %s' % + (settings, foreground)) + sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runserver %s %s' % + (settings, bind, foreground)) + sh('sleep 30') + settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=%s' % settings + else: + call_task('sync', options={'settings': settings}) live_server_option = '' info("Running the tests now...") @@ -933,8 +939,9 @@ def test_integration(options): else: success = True finally: - stop(options) - _reset() + if local: + stop(options) + _reset() if not success: sys.exit(1) @@ -963,17 +970,17 @@ def run_tests(options): call_task('test', options={'prefix': prefix}) elif integration_tests: if integration_upload_tests: - call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.upload.tests.integration'}) + call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.upload.tests.integration', 'local': local}) elif integration_monitoring_tests: - call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.monitoring.tests.integration'}) + call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.monitoring.tests.integration', 'local': local}) elif integration_csw_tests: call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.tests.csw', 'local': local}) elif integration_bdd_tests: call_task('test_bdd', options={'local': local}) elif integration_server_tests: - call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.geoserver.tests.integration'}) + call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.geoserver.tests.integration', 'local': local}) else: - call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.tests.integration'}) + call_task('test_integration', options={'prefix': prefix, 'name': 'geonode.tests.integration', 'local': local}) sh('flake8 geonode') diff --git a/scripts/spcgeonode/docker-compose.override.yml b/scripts/spcgeonode/docker-compose.override.yml index 5ec8aea1d7f..2009fe89d56 100644 --- a/scripts/spcgeonode/docker-compose.override.yml +++ b/scripts/spcgeonode/docker-compose.override.yml @@ -18,7 +18,7 @@ services: command: "uwsgi --chdir=/spcgeonode --module=geonode.wsgi --socket=:8000 --http=127.0.0.1:8001 --processes=5 --py-autoreload=2" celery: << : *default-common-django - command: 'celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=10 -n worker1@%h' + command: 'celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=2 -n worker1@%h' nginx: image: geonode/spcgeonode:nginx-3.1 diff --git a/scripts/spcgeonode/docker-compose.yml b/scripts/spcgeonode/docker-compose.yml index 3ac02608e05..686d9fbc67e 100644 --- a/scripts/spcgeonode/docker-compose.yml +++ b/scripts/spcgeonode/docker-compose.yml @@ -72,7 +72,7 @@ services: celery: << : *default-common-django entrypoint: [] - command: 'celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h' + command: 'celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=2 -n worker1@%h' # Nginx is serving django static and media files and proxies to django and geonode nginx: diff --git a/test_upload.sh b/test_upload.sh new file mode 100755 index 00000000000..8d1cda2b4a8 --- /dev/null +++ b/test_upload.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +export SITEURL=http://localhost:8001/ +export BACKEND=geonode.geoserver +export DOCKER_COMPOSE_VERSION=1.19.0 +export GEOSERVER_SERVER_URL=http://geoserver:8080/geoserver/ +export GEOSERVER_SERVER_PORT=8080 +export ON_TRAVIS=True +export TEST_RUNNER_KEEPDB=True +export TEST_RUN_INTEGRATION=True +export TEST_RUN_INTEGRATION_SERVER=False +export TEST_RUN_INTEGRATION_UPLOAD=True +export TEST_RUN_INTEGRATION_MONITORING=False +export TEST_RUN_INTEGRATION_CSW=False +export TEST_RUN_INTEGRATION_BDD=False +export MONITORING_ENABLED=False +export SESSION_EXPIRED_CONTROL_ENABLED=True +export ASYNC_SIGNALS=False +export DATABASE_URL=postgis://geonode:geonode@db:5432/geonode +export GEODATABASE_URL=postgis://geonode:geonode@db:5432/geonode_data +export DEFAULT_BACKEND_DATASTORE=datastore + +# echo "Initialize DB"; +# chmod +x scripts/misc/create_dbs_travis.sh; +# scripts/misc/create_dbs_travis.sh before_script; + +paver run_tests --coverage --local false \ No newline at end of file