Harmony 1495 - Reject request on submittal if EULA(s) not accepted

app/middleware/cmr-collection-reader.ts

@@ -1,9 +1,10 @@
 import { NextFunction } from 'express';
 import { harmonyCollections } from '../models/services';
 import { getVariablesForCollection, CmrCollection, getCollectionsByIds, getCollectionsByShortName, cmrApiConfig } from '../util/cmr';
-import { NotFoundError } from '../util/errors';
+import { ForbiddenError, NotFoundError, ServerError } from '../util/errors';
 import HarmonyRequest from '../models/harmony-request';
 import { listToText } from '../util/string';
+import { EdlUserEulaInfo, verifyUserEula } from '../util/edl-api';
 
 // CMR Collection IDs separated by delimiters of single "+" or single whitespace
 // (some clients may translate + to space)
@@ -25,6 +26,37 @@ async function loadVariablesForCollection(collection: CmrCollection, token: stri
   c.variables = await getVariablesForCollection(collection, token);
 }
 
+/**
+ * Check that the user has accepted any EULAs that are attached to the collections
+ * in the request.
+ * @param collections - array of CMR collections
+ * @param req - the client request
+ * @throws ServerError, ForbiddenError, NotFoundError
+ */
+async function verifyEulaAcceptance(collections: CmrCollection[], req: HarmonyRequest): Promise<void> {
+  const acceptEulaUrls = [];
+  for (const collection of collections) {
+    if (collection.eula_identifiers) {
+      for (const eulaId of collection.eula_identifiers) {
+        const eulaInfo: EdlUserEulaInfo = await verifyUserEula(req.user, eulaId, req.accessToken);
+        if (eulaInfo.statusCode == 404 && eulaInfo.acceptEulaUrl) { // EULA wasn't accepted
+          acceptEulaUrls.push(eulaInfo.acceptEulaUrl);  
+        } else if (eulaInfo.statusCode == 404) {
+          req.context.logger.error(`EULA (${eulaId}) verfification failed with statusCode 404. Error: ${eulaInfo.error}`);
+          throw new NotFoundError(`EULA ${eulaId} could not be found.`);
+        } else if (eulaInfo.statusCode !== 200) {
+          req.context.logger.error(`EULA (${eulaId}) verfification failed. Error: ${eulaInfo.error}`);
+          throw new ServerError(`EULA (${eulaId}) verfification failed unexpectedly.`);
+        }
+      }
+    }
+  }
+  if (acceptEulaUrls.length > 0) {
+    throw new ForbiddenError('You may access the requested data by resubmitting your request ' +
+      `after accepting the following EULA(s): ${acceptEulaUrls.join(', ')}.`);
+  }
+}
+
 /**
  * Express.js middleware that reads a list of slash-separated CMR Collection IDs
  * from a URL and adds two attributes to the req object:
@@ -64,6 +96,8 @@ async function cmrCollectionReader(req: HarmonyRequest, res, next: NextFunction)
       req.collections = await getCollectionsByIds(collectionIds, req.accessToken);
       const { collections } = req;
 
+      await verifyEulaAcceptance(collections, req);
+
       // Could not find a requested collection
       if (collections.length === 0) {
         const message = `${collectionIdStr} must be a collection short name or CMR collection`
@@ -98,6 +132,8 @@ async function cmrCollectionReader(req: HarmonyRequest, res, next: NextFunction)
           pickedCollection = harmonyCollection || pickedCollection;
         }
         if (pickedCollection) {
+          await verifyEulaAcceptance([pickedCollection], req);
+
           req.collections = [pickedCollection];
           req.collectionIds = [pickedCollection.id];
           await loadVariablesForCollection(pickedCollection, req.accessToken);

app/util/cmr.ts

@@ -64,6 +64,7 @@ export interface CmrCollection {
   };
   variables?: CmrUmmVariable[];
   tags?: CmrTags;
+  eula_identifiers?: string[];
 }
 
 export interface CmrGranule {

app/util/edl-api.ts

@@ -7,6 +7,8 @@ import HarmonyRequest from '../models/harmony-request';
 const edlUserRequestUrl = `${env.oauthHost}/oauth/tokens/user`;
 const edlClientCredentialsUrl = `${env.oauthHost}/oauth/token`;
 const edlUserGroupsBaseUrl = `${env.oauthHost}/api/user_groups/groups_for_user`;
+const edlVerifyUserEulaUrl = (username: string, eulaId: string): string =>
+  `${env.oauthHost}/api/users/${username}/verify_user_eula?eula_id=${eulaId}`;
 
 const clientCredentialsData = {
   params: { grant_type: 'client_credentials' },
@@ -123,4 +125,40 @@ export async function isAdminUser(req: HarmonyRequest): Promise<boolean> {
   const isAdmin = req.context.isAdminAccess ||
     (await getEdlGroupInformation(req.user, req.accessToken, req.context.logger)).isAdmin;
   return isAdmin;
+}
+
+export interface EdlUserEulaInfo {
+  statusCode: number;
+  error?: string;
+  acceptEulaUrl?: string;
+}
+
+/**
+ * Check whether the user has accepted a EULA.
+ *
+ * @param username - The EDL username
+ * @param eulaId - The id of the EULA (from the collection metadata)
+ * @param userToken - The user's token
+ * @returns A promise which resolves to info about whether the user has accepted a EULA,
+ * and if not, where they can go to accept it
+ */
+export async function verifyUserEula(username: string, eulaId: string, userToken: string)
+  : Promise<EdlUserEulaInfo> {
+  let statusCode: number;
+  let eulaResponse: { msg: string, error: string, accept_eula_url: string };
+  try {
+    const response = await axios.default.get(
+      edlVerifyUserEulaUrl(username, eulaId), { headers: { Authorization: `Bearer ${userToken}` } },
+    );
+    eulaResponse = response.data;
+    statusCode = response.status;
+  } catch (e) {
+    eulaResponse = e.response.data;
+    statusCode = e.response.status;
+  }
+  return {
+    statusCode,
+    error: eulaResponse.error,
+    acceptEulaUrl: eulaResponse.accept_eula_url,
+  };
 }

fixtures/cmr.uat.earthdata.nasa.gov-443/168970127878753735

@@ -0,0 +1,28 @@
+GET /search/collections.json?concept_id=C1258836670-EEDTEST&page_size=100
+accept: application/json
+accept-encoding: gzip,deflate
+
+HTTP/1.1 200 OK
+content-type: application/json;charset=utf-8
+transfer-encoding: chunked
+connection: close
+date: Tue, 18 Jul 2023 17:27:58 GMT
+x-frame-options: SAMEORIGIN
+access-control-allow-origin: *
+x-xss-protection: 1; mode=block
+cmr-request-id: 5f66ad93-3ad6-4175-8d12-baed9b732c54
+strict-transport-security: max-age=31536000
+cmr-search-after: ["harmony clone (with eulas) harmony example data","EEDTEST","eula-test-harmony_example","1",1258836670,2]
+cmr-hits: 1
+access-control-expose-headers: CMR-Hits, CMR-Request-Id, X-Request-Id, CMR-Scroll-Id, CMR-Search-After, CMR-Timed-Out, CMR-Shapefile-Original-Point-Count, CMR-Shapefile-Simplified-Point-Count
+x-content-type-options: nosniff
+cmr-took: 39
+x-request-id: zV2EOp8jXop75htCQhbHtk54oftz_-MdX_qU5uXNsm_Mgb4FjDE_TQ==
+vary: Accept-Encoding, User-Agent
+server: ServerTokens ProductOnly
+x-cache: Miss from cloudfront
+via: 1.1 35c803afef083002d824403342d4c62e.cloudfront.net (CloudFront)
+x-amz-cf-pop: EWR53-P1
+x-amz-cf-id: zV2EOp8jXop75htCQhbHtk54oftz_-MdX_qU5uXNsm_Mgb4FjDE_TQ==
+
+{"feed":{"updated":"2023-07-18T17:27:58.839Z","id":"https://cmr.uat.earthdata.nasa.gov:443/search/collections.json?concept_id=C1258836670-EEDTEST&page_size=100","title":"ECHO dataset metadata","entry":[{"cloud_hosted":false,"time_start":"1970-06-26T00:00:00.000Z","version_id":"1","updated":"2000-01-01T00:00:00.000Z","dataset_id":"Harmony clone (with EULAs) Harmony Example Data","has_spatial_subsetting":false,"has_transforms":false,"associations":{"variables":["V1258838719-EEDTEST","V1258838721-EEDTEST","V1258838723-EEDTEST","V1258838725-EEDTEST"]},"has_variables":true,"data_center":"EEDTEST","short_name":"eula-test-harmony_example","title":"Harmony clone (with EULAs) Harmony Example Data","summary":"    A collection to use in testing Harmony whose data have the following attributes, useful in testing current    and future Harmony work, ensuring that results come back correctly, and that metadata is preserved.    1. Granule data is very small, under 1MB, often well under 1MB    2. Data follows the outlines of the continents, making visual inspection easy    3. Data follows a consistent gradient, with hue differing slightly by granule in sequential temporal order from       blue/purple (oldest) to red (newest)    4. Files have color interpretation set, allowing for color output, if services preserve it    5. Files have 4 variables / bands    6. Data bands have descriptions set    7. Files have custom metadata, HARMONY_HEX_COLOR, set to the central hex value of the file's gradient, and HARMONY_HUE, set to the numeric hue (0.0-1.0)    8. File names contain useful information about their sequential order, color, and contents    9. Granules with global extents exist, as well as granules which cover each continent, with corresponding metadata    10. Granule temporal aligns to full days (UTC) starting with January 1, 2020, with each file name starting with the numeric        julian date  ","service_features":{"opendap":{"has_formats":false,"has_variables":false,"has_transforms":false,"has_spatial_subsetting":false,"has_temporal_subsetting":false},"esi":{"has_formats":false,"has_variables":false,"has_transforms":false,"has_spatial_subsetting":false,"has_temporal_subsetting":false},"harmony":{"has_formats":false,"has_variables":false,"has_transforms":false,"has_spatial_subsetting":false,"has_temporal_subsetting":false}},"orbit_parameters":{},"eula_identifiers":["be7c8c07-65f7-4e63-a81d-78dfa187870e","a5242e69-dc27-455c-b2bc-1991af58f719"],"id":"C1258836670-EEDTEST","has_formats":false,"consortiums":["GEOSS"],"original_format":"UMM_JSON","has_temporal_subsetting":false,"browse_flag":false,"platforms":[],"association_details":{"variables":[{"concept_id":"V1258838719-EEDTEST"},{"concept_id":"V1258838721-EEDTEST"},{"concept_id":"V1258838723-EEDTEST"},{"concept_id":"V1258838725-EEDTEST"}]},"online_access_flag":false}]}}

fixtures/cmr.uat.earthdata.nasa.gov-443/168971315358216063

@@ -0,0 +1,28 @@
+GET /search/collections.json?concept_id=C1258839703-EEDTEST&page_size=100
+accept: application/json
+accept-encoding: gzip,deflate
+
+HTTP/1.1 200 OK
+content-type: application/json;charset=utf-8
+transfer-encoding: chunked
+connection: close
+date: Tue, 18 Jul 2023 20:45:53 GMT
+x-frame-options: SAMEORIGIN
+access-control-allow-origin: *
+x-xss-protection: 1; mode=block
+cmr-request-id: 5930be20-636a-46c1-a29d-9dda1cfeeb88
+strict-transport-security: max-age=31536000
+cmr-search-after: ["harmony clone (with 1 eula) harmony example data","EEDTEST","1-eula-test-harmony_example","1",1258839703,1]
+cmr-hits: 1
+access-control-expose-headers: CMR-Hits, CMR-Request-Id, X-Request-Id, CMR-Scroll-Id, CMR-Search-After, CMR-Timed-Out, CMR-Shapefile-Original-Point-Count, CMR-Shapefile-Simplified-Point-Count
+x-content-type-options: nosniff
+cmr-took: 44
+x-request-id: 5p7x2kOyprsqrZaapCFRMpS8Gj_LVkXAAc-YG9eA6lQYkx_8n44Uow==
+vary: Accept-Encoding, User-Agent
+server: ServerTokens ProductOnly
+x-cache: Miss from cloudfront
+via: 1.1 35c803afef083002d824403342d4c62e.cloudfront.net (CloudFront)
+x-amz-cf-pop: EWR53-P1
+x-amz-cf-id: 5p7x2kOyprsqrZaapCFRMpS8Gj_LVkXAAc-YG9eA6lQYkx_8n44Uow==
+
+{"feed":{"updated":"2023-07-18T20:45:53.662Z","id":"https://cmr.uat.earthdata.nasa.gov:443/search/collections.json?concept_id=C1258839703-EEDTEST&page_size=100","title":"ECHO dataset metadata","entry":[{"cloud_hosted":false,"time_start":"1970-06-26T00:00:00.000Z","version_id":"1","updated":"2000-01-01T00:00:00.000Z","dataset_id":"Harmony clone (with 1 EULA) Harmony Example Data","has_spatial_subsetting":false,"has_transforms":false,"associations":{"variables":["V1258839717-EEDTEST","V1258839719-EEDTEST","V1258839721-EEDTEST","V1258839724-EEDTEST"],"services":["S1257851197-EEDTEST"]},"has_variables":true,"data_center":"EEDTEST","short_name":"1-eula-test-harmony_example","title":"Harmony clone (with 1 EULA) Harmony Example Data","summary":"    A collection to use in testing Harmony whose data have the following attributes, useful in testing current    and future Harmony work, ensuring that results come back correctly, and that metadata is preserved.    1. Granule data is very small, under 1MB, often well under 1MB    2. Data follows the outlines of the continents, making visual inspection easy    3. Data follows a consistent gradient, with hue differing slightly by granule in sequential temporal order from       blue/purple (oldest) to red (newest)    4. Files have color interpretation set, allowing for color output, if services preserve it    5. Files have 4 variables / bands    6. Data bands have descriptions set    7. Files have custom metadata, HARMONY_HEX_COLOR, set to the central hex value of the file's gradient, and HARMONY_HUE, set to the numeric hue (0.0-1.0)    8. File names contain useful information about their sequential order, color, and contents    9. Granules with global extents exist, as well as granules which cover each continent, with corresponding metadata    10. Granule temporal aligns to full days (UTC) starting with January 1, 2020, with each file name starting with the numeric        julian date  ","service_features":{"opendap":{"has_formats":false,"has_variables":false,"has_transforms":false,"has_spatial_subsetting":false,"has_temporal_subsetting":false},"esi":{"has_formats":false,"has_variables":false,"has_transforms":false,"has_spatial_subsetting":false,"has_temporal_subsetting":false},"harmony":{"has_formats":false,"has_variables":false,"has_transforms":false,"has_spatial_subsetting":false,"has_temporal_subsetting":false}},"orbit_parameters":{},"eula_identifiers":["be7c8c07-65f7-4e63-a81d-78dfa187870e"],"id":"C1258839703-EEDTEST","has_formats":false,"consortiums":["GEOSS"],"original_format":"UMM_JSON","has_temporal_subsetting":false,"browse_flag":false,"platforms":[],"association_details":{"variables":[{"concept_id":"V1258839717-EEDTEST"},{"concept_id":"V1258839719-EEDTEST"},{"concept_id":"V1258839721-EEDTEST"},{"concept_id":"V1258839724-EEDTEST"}],"services":[{"concept_id":"S1257851197-EEDTEST"}]},"online_access_flag":false}]}}

fixtures/cmr.uat.earthdata.nasa.gov-443/168971333195027832

@@ -0,0 +1,28 @@
+GET /search/collections.json?concept_id=C1258840703-EEDTEST&page_size=100
+accept: application/json
+accept-encoding: gzip,deflate
+
+HTTP/1.1 200 OK
+content-type: application/json;charset=utf-8
+transfer-encoding: chunked
+connection: close
+date: Tue, 18 Jul 2023 20:48:51 GMT
+x-frame-options: SAMEORIGIN
+access-control-allow-origin: *
+x-xss-protection: 1; mode=block
+cmr-request-id: e4b7bb31-72a8-41fa-ac6e-9dafd4565973
+strict-transport-security: max-age=31536000
+cmr-search-after: ["harmony clone (with bad eula id) harmony example data","EEDTEST","bad-eula-id-test-harmony_example","1",1258840703,1]
+cmr-hits: 1
+access-control-expose-headers: CMR-Hits, CMR-Request-Id, X-Request-Id, CMR-Scroll-Id, CMR-Search-After, CMR-Timed-Out, CMR-Shapefile-Original-Point-Count, CMR-Shapefile-Simplified-Point-Count
+x-content-type-options: nosniff
+cmr-took: 40
+x-request-id: i3xJtzcx8LiGILjMtJNd6n3O4cQYx5zGP_42DcHn24cJcd3Rm-T8UQ==
+vary: Accept-Encoding, User-Agent
+server: ServerTokens ProductOnly
+x-cache: Miss from cloudfront
+via: 1.1 aed3f8ed29085c056c75452d71b07f7e.cloudfront.net (CloudFront)
+x-amz-cf-pop: PHL50-C1
+x-amz-cf-id: i3xJtzcx8LiGILjMtJNd6n3O4cQYx5zGP_42DcHn24cJcd3Rm-T8UQ==
+
+{"feed":{"updated":"2023-07-18T20:48:51.989Z","id":"https://cmr.uat.earthdata.nasa.gov:443/search/collections.json?concept_id=C1258840703-EEDTEST&page_size=100","title":"ECHO dataset metadata","entry":[{"cloud_hosted":false,"time_start":"1970-06-26T00:00:00.000Z","version_id":"1","updated":"2000-01-01T00:00:00.000Z","dataset_id":"Harmony clone (with bad EULA id) Harmony Example Data","has_spatial_subsetting":false,"has_transforms":false,"associations":{"variables":["V1258840720-EEDTEST","V1258840714-EEDTEST","V1258840716-EEDTEST","V1258840718-EEDTEST"],"services":["S1257851197-EEDTEST"]},"has_variables":true,"data_center":"EEDTEST","short_name":"bad-eula-id-test-harmony_example","title":"Harmony clone (with bad EULA id) Harmony Example Data","summary":"    A collection to use in testing Harmony whose data have the following attributes, useful in testing current    and future Harmony work, ensuring that results come back correctly, and that metadata is preserved.    1. Granule data is very small, under 1MB, often well under 1MB    2. Data follows the outlines of the continents, making visual inspection easy    3. Data follows a consistent gradient, with hue differing slightly by granule in sequential temporal order from       blue/purple (oldest) to red (newest)    4. Files have color interpretation set, allowing for color output, if services preserve it    5. Files have 4 variables / bands    6. Data bands have descriptions set    7. Files have custom metadata, HARMONY_HEX_COLOR, set to the central hex value of the file's gradient, and HARMONY_HUE, set to the numeric hue (0.0-1.0)    8. File names contain useful information about their sequential order, color, and contents    9. Granules with global extents exist, as well as granules which cover each continent, with corresponding metadata    10. Granule temporal aligns to full days (UTC) starting with January 1, 2020, with each file name starting with the numeric        julian date  ","service_features":{"opendap":{"has_formats":false,"has_variables":false,"has_transforms":false,"has_spatial_subsetting":false,"has_temporal_subsetting":false},"esi":{"has_formats":false,"has_variables":false,"has_transforms":false,"has_spatial_subsetting":false,"has_temporal_subsetting":false},"harmony":{"has_formats":false,"has_variables":false,"has_transforms":false,"has_spatial_subsetting":false,"has_temporal_subsetting":false}},"orbit_parameters":{},"eula_identifiers":["be7c8c07-65f7-4e63-a81d-78dfa187879x"],"id":"C1258840703-EEDTEST","has_formats":false,"consortiums":["GEOSS"],"original_format":"UMM_JSON","has_temporal_subsetting":false,"browse_flag":false,"platforms":[],"association_details":{"variables":[{"concept_id":"V1258840720-EEDTEST"},{"concept_id":"V1258840714-EEDTEST"},{"concept_id":"V1258840716-EEDTEST"},{"concept_id":"V1258840718-EEDTEST"}],"services":[{"concept_id":"S1257851197-EEDTEST"}]},"online_access_flag":false}]}}