Merge pull request #1029 from chaoss/dev

Ready for Next Release

.github/CODEOWNERS

@@ -1,14 +1,16 @@
 /augur/cli/ @ccarterlandis
-/augur/metrics/ @ccarterlandis
+/augur/metrics/ @ccarterlandis @sgoggins
 /augur/housekeeper/ @gabe-heim
 /augur/server.py @ccarterlandis @gabe-heim
-/augur/application.py @ccarterlandis @gabe-heim
-/augur/routes/ @ccarterlandis @gabe-heim
+/augur/application.py @ccarterlandis @sgoggins @gabe-heim
+/augur/routes/ @ccarterlandis @gabe-heim @sgoggins
 
 /frontend/ @gabe-heim @sgoggins
 
 /schema/ @sgoggins @ccarterlandis
 
+/schema/generate @sgoggins 
+
 /workers/ @ccarterlandis @gabe-heim @sgoggins
 
 /README.md @sgoggins @ccarterlandis
@@ -27,4 +29,4 @@
 /Makefile @ccarterlandis
 /tox.ini @ccarterlandis
 *requirements.txt @ccarterlandis
-/setup.py @ccarterlandis
+/setup.py @ccarterlandis @sgoggins

.gitignore

@@ -154,3 +154,16 @@ workers/spdx_worker/dosocs2.conf
 workers/spdx_worker/augur-spdx/
 workers/spdx_worker/ex-raw.txt
 workers/spdx_worker/ex.json
+
+# Model files for machine learning
+workers/message_instights_worker/message_models/*.h5
+.h5
+*.h5
+workers/clustering_worker/kmeans_repo_messages
+workers/clustering_worker/lda_model
+workers/clustering_worker/vocabulary
+workers/clustering_worker/vocabulary_count
+
+
+# gzipped files
+*.gz

Makefile

@@ -144,14 +144,14 @@ docker-build-database:
 docker-run-backend:
 	@ - docker stop augur_backend
 	@ - docker rm augur_backend
-	@ docker run -p 5000:5000 --name augur_backend --env-file docker_env.txt augurlabs/augur:backend
+	docker run -p 5000:5000 --name augur_backend --env-file docker_env.txt augurlabs/augur:backend
 
 docker-run-frontend:
 	@ - docker stop augur_frontend
 	@ - docker rm augur_frontend
-	@ docker run -p 8080:8080 --name augur_frontend augurlabs/augur:frontend
+	docker run -p 8080:8080 --name augur_frontend augurlabs/augur:frontend
 
 docker-run-database:
 	@ - docker stop augur_database
 	@ - docker rm augur_database
-	@ docker run -p 5434:5432 --name augur_database augurlabs/augur:database
+	docker run -p 5434:5432 --name augur_database augurlabs/augur:database

augur/cli/backend.py

@@ -26,7 +26,9 @@ def cli():
 @cli.command("start")
 @click.option("--disable-housekeeper", is_flag=True, default=False, help="Turns off the housekeeper")
 @click.option("--skip-cleanup", is_flag=True, default=False, help="Disables the old process cleanup that runs before Augur starts")
-def start(disable_housekeeper, skip_cleanup):
+@click.option("--logstash", is_flag=True, default=False, help="Runs logstash to collect errors from logs")
+@click.option("--logstash-with-cleanup", is_flag=True, default=False, help="Runs logstash to collect errors from logs and cleans all previously collected errors")
+def start(disable_housekeeper, skip_cleanup, logstash, logstash_with_cleanup):
     """
     Start Augur's backend server
     """
@@ -40,11 +42,26 @@ def start(disable_housekeeper, skip_cleanup):
     else:
         logger.debug("Skipping process cleanup")
 
+    if logstash or logstash_with_cleanup:
+        augur_home = os.getenv('ROOT_AUGUR_DIRECTORY', "")
+        if logstash_with_cleanup:
+            print("Cleaning old workers errors...")
+            with open(augur_home + "/log_analysis/http/empty_index.html") as f:
+                lines = f.readlines()
+            with open(augur_home + "/log_analysis/http/index.html", "w") as f1:
+                f1.writelines(lines)
+            print("All previous workers errors got deleted.")
+
+        elasticsearch_path = os.getenv('ELASTIC_SEARCH_PATH', "/usr/local/bin/elasticsearch")
+        subprocess.Popen(elasticsearch_path)
+        logstash_path = os.getenv('LOGSTASH_PATH', "/usr/local/bin/logstash")
+        subprocess.Popen([logstash_path, "-f", augur_home + "/log_analysis/logstash-filter.conf"])
+
     master = initialize_components(augur_app, disable_housekeeper)
 
     logger.info('Starting Gunicorn webserver...')
-    logger.info(f"Augur is running at: http://0.0.0.0:5000")
-    logger.info("Gunicorn server logs & errors will be written to logs/gunicorn.log")
+    logger.info(f'Augur is running at: http://127.0.0.1:{augur_app.config.get_value("Server", "port")}')
+    logger.info('Gunicorn server logs & errors will be written to logs/gunicorn.log')
     logger.info('Housekeeper update process logs will now take over.')
     Arbiter(master).run()