HARMONY-1499: Added ability to use conditional step in services.yml based on native file format info in umm-c.

app/middleware/cmr-umm-collection-reader.ts

@@ -0,0 +1,26 @@
+import { NextFunction } from 'express';
+import { getUmmCollectionsByIds } from '../util/cmr';
+import HarmonyRequest from '../models/harmony-request';
+
+/**
+ * Express.js middleware that reads the UMM JSON format of the collections and load them into operation
+ *
+ * @param req - The client request
+ * @param res - The client response
+ * @param next - The next function in the middleware chain
+ */
+async function cmrUmmCollectionReader(req: HarmonyRequest, res, next: NextFunction): Promise<void> {
+  try {
+    const hasUmmConditional = req.context.serviceConfig?.steps?.filter((s) => s.conditional?.umm_c);
+    if (hasUmmConditional && hasUmmConditional.length > 0) {
+      req.operation.ummcollections = await getUmmCollectionsByIds(req.collectionIds, req.accessToken);
+    }
+    next();
+  } catch (error) {
+    req.collectionIds = [];
+    req.collections = [];
+    next(error);
+  }
+}
+
+export = cmrUmmCollectionReader;

app/models/data-operation.ts

@@ -4,7 +4,7 @@ import Ajv from 'ajv';
 import addFormats from 'ajv-formats';
 import _ from 'lodash';
 import logger from '../util/log';
-import { CmrUmmVariable } from '../util/cmr';
+import { CmrUmmCollection, CmrUmmVariable } from '../util/cmr';
 import { Encrypter, Decrypter } from '../util/crypto';
 import { cmrVarToHarmonyVar, HarmonyVariable } from '../util/variables';
 
@@ -852,6 +852,24 @@ export default class DataOperation {
     this.model.stagingLocation = value;
   }
 
+  /**
+   * Gets the umm collections for data produced by this request
+   *
+   * @returns the umm collections
+   */
+  get ummcollections(): CmrUmmCollection[] {
+    return this.model.ummcollections;
+  }
+
+  /**
+   * Sets the umm collections for data produced by this request
+   *
+   * @param value - the umm collections
+   */
+  set ummcollections(value: CmrUmmCollection[]) {
+    this.model.ummcollections = value;
+  }
+
   /**
    *  Returns a deep copy of this operation
    *
@@ -878,6 +896,7 @@ export default class DataOperation {
     }
 
     let toWrite = _.cloneDeep(this.model);
+    delete toWrite.ummcollections;
 
     // To be fixed by HARMONY-203 to not default to TIFF
     toWrite.format.mime = toWrite.format.mime || 'image/tiff';

app/models/services/base-service.ts

@@ -40,6 +40,9 @@ export interface ServiceStep {
   conditional?: {
     exists?: string[];
     format?: string[];
+    umm_c?: {
+      native_format?: string[];
+    }
   };
 }
 
@@ -159,6 +162,16 @@ function stepRequired(step: ServiceStep, operation: DataOperation): boolean {
       required = true;
     }
   }
+  if (required && step.conditional?.umm_c) {
+    required = false;
+    if (step.conditional.umm_c.native_format) {
+      const fileArchiveInfo = operation.ummcollections[0].umm.ArchiveAndDistributionInformation?.FileArchiveInformation;
+      const nativeFormat = fileArchiveInfo?.filter((a) => a.FormatType = 'Native')[0]?.Format;
+      if (nativeFormat && step.conditional.umm_c.native_format.includes(nativeFormat.toLowerCase())) {
+        required = true;
+      }
+    }
+  }
   return required;
 }
 

app/routers/router.ts

@@ -30,6 +30,7 @@ import serviceInvoker from '../backends/service-invoker';
 import HarmonyRequest, { addRequestContextToOperation } from '../models/harmony-request';
 
 import cmrCollectionReader = require('../middleware/cmr-collection-reader');
+import cmrUmmCollectionReader = require('../middleware/cmr-umm-collection-reader');
 import envVars = require('../util/env');
 import { postServiceConcatenationHandler, preServiceConcatenationHandler } from '../middleware/concatenation';
 import getRequestMetrics from '../frontends/request-metrics';
@@ -194,6 +195,7 @@ export default function router({ skipEarthdataLogin = 'false' }: RouterConfig):
   result.use(logged(preServiceConcatenationHandler));
   result.use(logged(chooseService));
   result.use(logged(postServiceConcatenationHandler));
+  result.use(logged(cmrUmmCollectionReader));
   result.use(logged(cmrGranuleLocator));
   result.use(logged(addRequestContextToOperation));
   result.use(logged(redirectWithoutTrailingSlash));

app/util/cmr.ts

@@ -254,6 +254,22 @@ export interface CmrPermissionsResponse extends CmrResponse {
   data: CmrPermissionsMap;
 }
 
+export interface CmrUmmCollection {
+  meta: {
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    'concept-id': string;
+  };
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  umm: any;
+}
+
+export interface CmrUmmCollectionsResponse extends CmrResponse {
+  data: {
+    items: CmrUmmCollection[];
+    hits: number;
+  };
+}
+
 /**
  * Create a token header for the given access token string
  *
@@ -594,6 +610,20 @@ async function queryCollections(
   return collectionsResponse.data.feed.entry;
 }
 
+/**
+ * Performs a CMR collections.umm_json search with the given query string
+ *
+ * @param query - The key/value pairs to search
+ * @param token - Access token for user request
+ * @returns The umm collection search results
+ */
+async function queryUmmCollections(
+  query: CmrQuery, token: string,
+): Promise<Array<CmrUmmCollection>> {
+  const ummResponse = await _cmrGet('/search/collections.umm_json', query, token) as CmrUmmCollectionsResponse;
+  return ummResponse.data.items;
+}
+
 /**
  * Performs a CMR grids.umm_json search with the given query string
  *
@@ -655,6 +685,25 @@ export function getCollectionsByIds(
   return queryCollections(query, token);
 }
 
+/**
+ * Queries and returns the CMR UMM JSON collections corresponding to the given CMR Collection IDs
+ *
+ * @param ids - The collection IDs to find
+ * @param token - Access token for user request
+ * @param includeTags - Include tags with tag_key matching this value
+ * @returns The umm collections with the given ids
+ */
+export function getUmmCollectionsByIds(
+  ids: Array<string>,
+  token: string,
+): Promise<Array<CmrUmmCollection>> {
+  const query = {
+    concept_id: ids,
+    page_size: cmrMaxPageSize,
+  };
+  return queryUmmCollections(query, token);
+}
+
 /**
  * Queries and returns the CMR JSON collections corresponding to the given collection short names
  *

test/models/services.ts

@@ -812,6 +812,19 @@ describe('createWorkflowSteps', function () {
   const versionId = '1';
   const operation = buildOperation('foo');
   operation.addSource(collectionId, shortName, versionId);
+  // the existence of conditional umm_c in config guarantees that ummCollections is set
+  operation.ummcollections = [{
+    'meta': {
+      'concept-id': 'C1234-TEST',
+    },
+    'umm': {
+      'MetadataSpecification': {
+        'Name': 'UMM-C',
+        'URL': 'https://cdn.earthdata.nasa.gov/umm/collection/v1.17.3',
+        'Version': '1.17.3',
+      },
+    },
+  }];
   const config = {
     name: 'shapefile-tiff-netcdf-service',
     data_operation_version: CURRENT_SCHEMA_VERSION,
@@ -825,6 +838,14 @@ describe('createWorkflowSteps', function () {
     },
     steps: [{
       image: 'query cmr',
+    }, {
+      image: 'format transformer',
+      operations: ['formatTransform'],
+      conditional: {
+        umm_c: {
+          native_format: ['netcdf-4'],
+        },
+      },
     }, {
       image: 'temporal subsetter',
       operations: ['temporalSubset'],
@@ -1033,4 +1054,54 @@ describe('createWorkflowSteps', function () {
       expect(stagingLocation).to.include('dummy/p1');
     });
   });
+
+  describe('when a collection has matching umm-c conditional native_format', function () {
+    const ummOperation = _.cloneDeep(operation);
+    ummOperation.geojson = 'interesting shape';
+    ummOperation.ummcollections = [{
+      'meta': {
+        'concept-id': 'C1234-TEST',
+      },
+      'umm': {
+        'MetadataSpecification': {
+          'Name': 'UMM-C',
+          'URL': 'https://cdn.earthdata.nasa.gov/umm/collection/v1.17.3',
+          'Version': '1.17.3',
+        },
+        'ArchiveAndDistributionInformation': {
+          'FileArchiveInformation': [ {
+            'Format': 'netCDF-4',
+            'FormatType': 'Native',
+          } ],
+          'FileDistributionInformation': [ {
+            'Format': 'netCDF-4',
+            'FormatType': 'Native',
+          } ],
+        },
+      },
+    }];
+    const service = new StubService(config, {}, ummOperation);
+    const steps = service.createWorkflowSteps();
+
+    it('is not synchronous', function () {
+      expect(service.isSynchronous).to.equal(false);
+      expect(service.operation.isSynchronous).to.equal(false);
+    });
+
+    it('creates three workflow steps', function () {
+      expect(steps.length).to.equal(3);
+    });
+
+    it('creates a first workflow step for query cmr', function () {
+      expect(steps[0].serviceID).to.equal('query cmr');
+    });
+
+    it('creates a second step for conditional umm_c native_format', function () {
+      expect(steps[1].serviceID).to.equal('format transformer');
+    });
+
+    it('creates a third and final workflow step for the shapefile subsetter', function () {
+      expect(steps[2].serviceID).to.equal('shapefile subsetter');
+    });
+  });
 });