Release-6.0.1 (#2153)

* docker cleanup cronjob flag & variable * release v6.0.1 * increase scroll duration to fix reconciliation specs * Update tf-modules/cumulus/variables.tf Co-authored-by: Jonathan Kovarik <[email protected]> * backport note * Changes waitForAsyncOperationStatus to fail if status not reached. (#2125) Co-authored-by: Jonathan Kovarik <[email protected]> Co-authored-by: Matt Savoie <[email protected]>
nasa · Mar 25, 2021 · 6a3a3d7 · 6a3a3d7
1 parent 4617ff8
commit 6a3a3d7
Show file tree

Hide file tree

Showing 65 changed files with 340 additions and 269 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+## [v6.0.1] 2021-03-23
+
+### MIGRATION NOTES
+
+Cumulus 6.0.1 is a backport release to address urgent, specific issues faced by a DAAC with the Cumulus ECS Cluster.
+This fix will not be carried forward in the same form, instead a similar but more useful and configurable change will be worked on the forward release builds.
+
+### Added
+
+- **CUMULUS-2421**
+  - Added cron job to clean up docker storage blocks to ECS cluster template.
+  - Added `ecs_include_docker_cleanup_cronjob` boolean variable to the Cumulus module to enable this behavior. Default value is `false`.
+
 ## [v6.0.0] 2021-02-16
 
 ### MIGRATION NOTES
@@ -3772,6 +3785,7 @@ Note: There was an issue publishing 1.12.0. Upgrade to 1.12.1.
 ## [v1.0.0] - 2018-02-23
 
 [unreleased]: https://github.com/nasa/cumulus/compare/v6.0.0...HEAD
+[v6.0.1]: https://github.com/nasa/cumulus/compare/v6.0.0...v6.0.1
 [v6.0.0]: https://github.com/nasa/cumulus/compare/v5.0.1...v6.0.0
 [v5.0.1]: https://github.com/nasa/cumulus/compare/v5.0.0...v5.0.1
 [v5.0.0]: https://github.com/nasa/cumulus/compare/v4.0.0...v5.0.0

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
@@ -31,3 +31,4 @@
 * Phil Osip
 * Sara Chaparro Diaz
 * Seth Vincent
+* Snyk bot
diff --git a/example/cumulus-tf/main.tf b/example/cumulus-tf/main.tf
@@ -75,6 +75,7 @@ module "cumulus" {
   ecs_cluster_min_size            = 2
   ecs_cluster_desired_size        = 2
   ecs_cluster_max_size            = 3
+  ecs_include_docker_cleanup_cronjob = var.ecs_include_docker_cleanup_cronjob
   key_name                        = var.key_name
   ecs_custom_sg_ids               = var.ecs_custom_sg_ids
 

diff --git a/example/cumulus-tf/variables.tf b/example/cumulus-tf/variables.tf
@@ -188,6 +188,11 @@ variable "ecs_cluster_instance_subnet_ids" {
   default = []
 }
 
+variable "ecs_include_docker_cleanup_cronjob" {
+  type    = bool
+  default = false
+}
+
 variable "ems_datasource" {
   type        = string
   description = "the data source of EMS reports"

diff --git a/example/lambdas/asyncOperations/package.json b/example/lambdas/asyncOperations/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-async-operations",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "AsyncOperations Test Lambda",
   "main": "index.js",
   "private": true,

diff --git a/example/lambdas/python-processing/package.json b/example/lambdas/python-processing/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@cumulus/python-process-activity",
   "private": true,
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "Python reference activity",
   "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity",
   "repository": {

diff --git a/example/lambdas/python-reference-activity/package.json b/example/lambdas/python-reference-activity/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@cumulus/python-reference-activity",
   "private": true,
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "Python reference activity",
   "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-activity",
   "repository": {

diff --git a/example/lambdas/python-reference-task/package.json b/example/lambdas/python-reference-task/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@cumulus/python-reference-task",
   "private": true,
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "Python reference task",
   "main": "index.js",
   "homepage": "https://github.com/nasa/cumulus/tree/master/example/lambdas/python-reference-task",

diff --git a/example/lambdas/s3AccessTest/package.json b/example/lambdas/s3AccessTest/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-s3-access",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "S3 Access Test Lambda",
   "main": "index.js",
   "private": true,

diff --git a/example/lambdas/snsS3Test/package.json b/example/lambdas/snsS3Test/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-sns-s3",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "SNS to S3 Test Lambda",
   "main": "index.js",
   "private": true,

diff --git a/example/lambdas/versionUpTest/package.json b/example/lambdas/versionUpTest/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/test-version-up",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "Version Up Test Lambda",
   "main": "index.js",
   "private": true,

diff --git a/example/package.json b/example/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/cumulus-integration-tests",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "Cumulus Integration Test Deployment",
   "private": true,
   "main": "index.js",
@@ -44,30 +44,30 @@
     ]
   },
   "dependencies": {
-    "@cumulus/api": "6.0.0",
-    "@cumulus/api-client": "6.0.0",
-    "@cumulus/aws-client": "6.0.0",
-    "@cumulus/checksum": "6.0.0",
-    "@cumulus/cmr-client": "6.0.0",
-    "@cumulus/cmrjs": "6.0.0",
-    "@cumulus/common": "6.0.0",
-    "@cumulus/discover-granules": "6.0.0",
-    "@cumulus/discover-pdrs": "6.0.0",
-    "@cumulus/files-to-granules": "6.0.0",
-    "@cumulus/hello-world": "6.0.0",
-    "@cumulus/integration-tests": "6.0.0",
-    "@cumulus/message": "6.0.0",
-    "@cumulus/move-granules": "6.0.0",
-    "@cumulus/parse-pdr": "6.0.0",
-    "@cumulus/pdr-status-check": "6.0.0",
-    "@cumulus/post-to-cmr": "6.0.0",
-    "@cumulus/queue-granules": "6.0.0",
-    "@cumulus/queue-pdrs": "6.0.0",
-    "@cumulus/sf-sqs-report": "6.0.0",
-    "@cumulus/sync-granule": "6.0.0",
-    "@cumulus/test-processing": "6.0.0"
+    "@cumulus/api": "6.0.1",
+    "@cumulus/api-client": "6.0.1",
+    "@cumulus/aws-client": "6.0.1",
+    "@cumulus/checksum": "6.0.1",
+    "@cumulus/cmr-client": "6.0.1",
+    "@cumulus/cmrjs": "6.0.1",
+    "@cumulus/common": "6.0.1",
+    "@cumulus/discover-granules": "6.0.1",
+    "@cumulus/discover-pdrs": "6.0.1",
+    "@cumulus/files-to-granules": "6.0.1",
+    "@cumulus/hello-world": "6.0.1",
+    "@cumulus/integration-tests": "6.0.1",
+    "@cumulus/message": "6.0.1",
+    "@cumulus/move-granules": "6.0.1",
+    "@cumulus/parse-pdr": "6.0.1",
+    "@cumulus/pdr-status-check": "6.0.1",
+    "@cumulus/post-to-cmr": "6.0.1",
+    "@cumulus/queue-granules": "6.0.1",
+    "@cumulus/queue-pdrs": "6.0.1",
+    "@cumulus/sf-sqs-report": "6.0.1",
+    "@cumulus/sync-granule": "6.0.1",
+    "@cumulus/test-processing": "6.0.1"
   },
   "devDependencies": {
-    "@cumulus/test-data": "6.0.0"
+    "@cumulus/test-data": "6.0.1"
   }
 }
diff --git a/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js b/example/spec/parallel/createReconciliationReport/CreateReconciliationReportSpec.js
@@ -424,13 +424,21 @@ describe('When there are granule differences and granule reconciliation is run',
     });
 
     it('generates reconciliation report through the Cumulus API', async () => {
-      const asyncOperation = await waitForAsyncOperationStatus({
-        id: asyncOperationId,
-        status: 'SUCCEEDED',
-        stackName: config.stackName,
-        retries: 100,
-      });
-
+      let asyncOperation;
+      try {
+        asyncOperation = await waitForAsyncOperationStatus({
+          id: asyncOperationId,
+          status: 'SUCCEEDED',
+          stackName: config.stackName,
+          retryOptions: {
+            retries: 70,
+            factor: 1.041,
+          },
+        });
+      } catch (error) {
+        fail(error);
+      }
+      expect(asyncOperation.status).toEqual('SUCCEEDED');
       reportRecord = JSON.parse(asyncOperation.output);
     });
 
@@ -563,13 +571,20 @@ describe('When there are granule differences and granule reconciliation is run',
     });
 
     it('generates reconciliation report through the Cumulus API', async () => {
-      const asyncOperation = await waitForAsyncOperationStatus({
-        id: asyncOperationId,
-        status: 'SUCCEEDED',
-        stackName: config.stackName,
-        retries: 100,
-      });
-
+      let asyncOperation;
+      try {
+        asyncOperation = await waitForAsyncOperationStatus({
+          id: asyncOperationId,
+          status: 'SUCCEEDED',
+          stackName: config.stackName,
+          retryOptions: {
+            retries: 70,
+            factor: 1.041,
+          },
+        });
+      } catch (error) {
+        fail(error);
+      }
       reportRecord = JSON.parse(asyncOperation.output);
     });
 
@@ -641,12 +656,20 @@ describe('When there are granule differences and granule reconciliation is run',
     });
 
     it('generates reconciliation report through the Cumulus API', async () => {
-      const asyncOperation = await waitForAsyncOperationStatus({
-        id: asyncOperationId,
-        status: 'SUCCEEDED',
-        stackName: config.stackName,
-        retries: 100,
-      });
+      let asyncOperation;
+      try {
+        asyncOperation = await waitForAsyncOperationStatus({
+          id: asyncOperationId,
+          status: 'SUCCEEDED',
+          stackName: config.stackName,
+          retryOptions: {
+            retries: 70,
+            factor: 1.041,
+          },
+        });
+      } catch (error) {
+        fail(error);
+      }
 
       reportRecord = JSON.parse(asyncOperation.output);
     });

diff --git a/example/spec/parallel/ingestGranule/IngestGranuleSuccessSpec.js b/example/spec/parallel/ingestGranule/IngestGranuleSuccessSpec.js
@@ -777,7 +777,10 @@ describe('The S3 Ingest Granules workflow', () => {
             id: asyncOperationId,
             status: 'SUCCEEDED',
             stackName: config.stackName,
-            retries: 100,
+            retryOptions: {
+              retries: 70,
+              factor: 1.041,
+            },
           });
 
           const reingestOutput = JSON.parse(asyncOperation.output);

diff --git a/lambdas/sqs-message-remover/package.json b/lambdas/sqs-message-remover/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/sqs-message-remover-lambda",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "Remove processed messages from SQS queues",
   "main": "src/index.js",
   "private": true,
@@ -35,9 +35,9 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/aws-client": "6.0.0",
-    "@cumulus/common": "6.0.0",
-    "@cumulus/logger": "6.0.0",
+    "@cumulus/aws-client": "6.0.1",
+    "@cumulus/common": "6.0.1",
+    "@cumulus/logger": "6.0.1",
     "lodash": "^4.17.20"
   }
 }
diff --git a/lerna.json b/lerna.json
@@ -1,6 +1,6 @@
 {
   "lerna": "3.20.2",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "packages": [
     "example",
     "example/lambdas/*",

diff --git a/packages/api-client/package.json b/packages/api-client/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/api-client",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "API client for working with the Cumulus archive API",
   "keywords": [
     "GIBS",
@@ -38,11 +38,11 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/aws-client": "6.0.0",
-    "@cumulus/logger": "6.0.0",
+    "@cumulus/aws-client": "6.0.1",
+    "@cumulus/logger": "6.0.1",
     "p-retry": "^2.0.0"
   },
   "devDependencies": {
-    "@cumulus/types": "6.0.0"
+    "@cumulus/types": "6.0.1"
   }
 }
diff --git a/packages/api/es/esDefaults.js b/packages/api/es/esDefaults.js
@@ -1,5 +1,5 @@
 const defaultESScrollSize = 1000;
-const defaultESScrollDuration = '2m';
+const defaultESScrollDuration = '6m';
 
 module.exports = {
   defaultESScrollSize,

diff --git a/packages/api/package.json b/packages/api/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/api",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "Lambda functions for handling all daac's API operations",
   "main": "index.js",
   "engines": {
@@ -45,19 +45,19 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/aws-client": "6.0.0",
-    "@cumulus/cmr-client": "6.0.0",
-    "@cumulus/cmrjs": "6.0.0",
-    "@cumulus/collection-config-store": "6.0.0",
-    "@cumulus/common": "6.0.0",
-    "@cumulus/earthdata-login-client": "6.0.0",
-    "@cumulus/errors": "6.0.0",
-    "@cumulus/ingest": "6.0.0",
-    "@cumulus/launchpad-auth": "6.0.0",
-    "@cumulus/logger": "6.0.0",
-    "@cumulus/message": "6.0.0",
-    "@cumulus/pvl": "6.0.0",
-    "@cumulus/sftp-client": "6.0.0",
+    "@cumulus/aws-client": "6.0.1",
+    "@cumulus/cmr-client": "6.0.1",
+    "@cumulus/cmrjs": "6.0.1",
+    "@cumulus/collection-config-store": "6.0.1",
+    "@cumulus/common": "6.0.1",
+    "@cumulus/earthdata-login-client": "6.0.1",
+    "@cumulus/errors": "6.0.1",
+    "@cumulus/ingest": "6.0.1",
+    "@cumulus/launchpad-auth": "6.0.1",
+    "@cumulus/logger": "6.0.1",
+    "@cumulus/message": "6.0.1",
+    "@cumulus/pvl": "6.0.1",
+    "@cumulus/sftp-client": "6.0.1",
     "@elastic/elasticsearch": "^5.6.20",
     "@mapbox/dyno": "^1.4.2",
     "aggregate-error": "^3.0.1",
@@ -98,6 +98,6 @@
     "xml2js": "^0.4.22"
   },
   "devDependencies": {
-    "@cumulus/test-data": "6.0.0"
+    "@cumulus/test-data": "6.0.1"
   }
 }
diff --git a/packages/aws-client/package.json b/packages/aws-client/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/aws-client",
-  "version": "6.0.0",
+  "version": "6.0.1",
   "description": "Utilities for working with AWS",
   "keywords": [
     "GIBS",
@@ -43,9 +43,9 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/checksum": "6.0.0",
-    "@cumulus/errors": "6.0.0",
-    "@cumulus/logger": "6.0.0",
+    "@cumulus/checksum": "6.0.1",
+    "@cumulus/errors": "6.0.1",
+    "@cumulus/logger": "6.0.1",
     "aws-sdk": "^2.585.0",
     "jsonpath-plus": "^1.1.0",
     "lodash": "~4.17.20",