DHFPROD-9448: Temporal feature

.editorconfig

@@ -16,6 +16,9 @@ max_line_length = 120
 [*.sjs]
 indent_size = 2
 
+[*.mjs]
+indent_size = 2
+
 [*.js]
 indent_size = 2
 

Jenkinsfile

@@ -353,6 +353,7 @@ void dh5Example() {
                             rm -rf $WORKSPACE$GRADLE_DIR/caches; \
                             ./gradlew -i hubInit -Ptesting=true; \
                             cp ../../marklogic-data-hub/gradle.properties .; \
+                            ./gradlew mlSetTraceEvents -Pevents=hub-core
                             ./gradlew -i mlDeploy -Ptesting=true -PmlUsername=admin -PmlPassword=admin; \
                             ./gradlew hubRunFlow -PflowName=ingestion_only-flow -Ptesting=true; \
                             ./gradlew hubRunFlow -PflowName=ingestion_mapping-flow -Ptesting=true; \

marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/createAxis.json

@@ -0,0 +1,6 @@
+{
+  "local-name" : "createAxis",
+  "document-uri" : "/data-hub/5/temporal/hub-temporal.mjs",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "temporal-admin" ]
+}

marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/createCollection.json

@@ -0,0 +1,6 @@
+{
+  "local-name" : "createCollection",
+  "document-uri" : "/data-hub/5/temporal/hub-temporal.mjs",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "temporal-admin" ]
+}

marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/createIndex.json

@@ -0,0 +1,6 @@
+{
+  "local-name" : "createIndex",
+  "document-uri" : "/data-hub/5/temporal/hub-temporal.mjs",
+  "modules-database" : "%%mlModulesDbName%%",
+  "role" : [ "temporal-admin" ]
+}

marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/temporal/hub-temporal.mjs

@@ -23,7 +23,42 @@ function getTemporalCollections() {
   return temporal.collections();
 }
 
+function createIndex() {
+  xdmp.invokeFunction(function () {
+    const admin = require("/MarkLogic/admin");
+    let config = admin.getConfiguration();
+    let elementRangeIndexes = [
+      admin.databaseRangeElementIndex("dateTime", "", "systemStart", "", fn.false()),
+      admin.databaseRangeElementIndex("dateTime", "", "systemEnd", "", fn.false())]
+    elementRangeIndexes.forEach((elementRangeIndex) => {
+      try {
+        config = admin.databaseAddRangeElementIndex(config, xdmp.database(), elementRangeIndex);
+      } catch (e) {
+      }
+    });
+    admin.saveConfiguration(config);
+  },{update: "true"});
+}
+
+function createAxis() {
+  xdmp.invokeFunction(function () {
+    try {
+      temporal.axisCreate("system", cts.elementReference("systemStart", "type=dateTime"), cts.elementReference("systemEnd", "type=dateTime"));
+    } catch (e) {
+    }
+  }, {update: "true"});
+}
+
+function createCollection(temporalCollection) {
+  xdmp.invokeFunction(function () {
+    temporal.collectionCreate(temporalCollection, "system");
+  }, {update: "true"});
+}
+
 export default {
-    getTemporalCollections: import.meta.amp(getTemporalCollections)
+  getTemporalCollections: import.meta.amp(getTemporalCollections),
+  createIndex: import.meta.amp(createIndex),
+  createAxis: import.meta.amp(createAxis),
+  createCollection: import.meta.amp(createCollection)
 }
 

marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/features/core.mjs

@@ -43,25 +43,7 @@ function getFeatureMethod(featureName, featureMethod) {
   return registeredFeatures[featureName][featureMethod];
 }
 
-function invokeFeatureMethods(stepExecutionContext, contentArray, method) {
-  const flowStep = stepExecutionContext.flowStep;
-  let targetEntityType = flowStep.options.targetEntity || flowStep.options.targetEntityType;
-  let model = null;
-  if (targetEntityType) {
-    const modelNode = entityLib.findModelForEntityTypeId(targetEntityType);
-    model = fn.exists(modelNode) ? modelNode.toObject() : null;
-  }
-  const features = Object.keys(featuresCore.getFeatures());
-  features.forEach(feat => {
-    const funct = featuresCore.getFeatureMethod(feat, method);
-    if (funct) {
-      funct(stepExecutionContext, model, contentArray);
-    }
-  });
-}
-
 export default {
   getFeatures,
-  getFeatureMethod,
-  invokeFeatureMethods
+  getFeatureMethod
 };

marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/features/doc-permissions.mjs

@@ -21,6 +21,7 @@
 
 import consts from "/data-hub/5/impl/consts.mjs";
 import hubUtils from "/data-hub/5/impl/hub-utils.mjs";
+import featuresUtils from "./features-util.mjs";
 
 const INFO_EVENT = consts.TRACE_CORE;
 
@@ -35,43 +36,23 @@ function onArtifactPublish(artifactType, artifactName) {
 }
 
 function onInstanceSave(context, model, contentArray) {
-    let flagAddPermissionObject = false;
     let permissions = [];
     const stepContext = context.flowStep;
     if (!model) {
         return contentArray;
     }
-    const modelName = model.info.title;
-    if(featureEnabled(model.definitions[modelName])) {
-        hubUtils.hubTrace(INFO_EVENT, `Processing doc permission feature for an instance of ${modelName}`);
-        const docPermissionObj = model.definitions[modelName].features.find(item => item['docPermission']);
-        permissions = docPermissionObj['docPermission'].permissions;
-        flagAddPermissionObject = true;
-    }
-    if(featureEnabled(stepContext)) {
-        hubUtils.hubTrace(INFO_EVENT, `Processing doc permission feature for an instance while running ${stepContext.name}`);
-        const docPermissionObj = stepContext.features.find(item => item['docPermission']);
 
-        permissions = docPermissionObj["docPermission"].permissions;
-        flagAddPermissionObject = true;
-    }
-    if (flagAddPermissionObject){
-        addPermissionsToObject(permissions, contentArray);
+    const feature = featuresUtils.getFeatureFromContext(stepContext, model, 'docPermission');
+    if(feature) {
+      const modelName = model.info.title;
+      hubUtils.hubTrace(INFO_EVENT, `Processing doc permission feature for an instance of ${modelName} while running ${stepContext.name}`);
+      permissions = feature.permissions;
+      addPermissionsToObject(permissions, contentArray);
     }
     hubUtils.hubTrace(INFO_EVENT, `Finished processing doc permission feature `);
     return contentArray;
 }
 
-function featureEnabled(artifact) {
-   if (artifact.features) {
-       const docPermissionObj = artifact.features.find(item => item['docPermission']);
-       if (docPermissionObj) {
-           return docPermissionObj['docPermission'].enabled;
-       }
-   }
-    return false;
-}
-
 function addPermissionsToObject(permissions, contentArray) {
     // add permissions to the contentObject
     contentArray.forEach(contentObject => {

marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/features/features-util.mjs

@@ -1,9 +1,12 @@
 /**
  Copyright (c) 2021 MarkLogic Corporation
+
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
+
  http://www.apache.org/licenses/LICENSE-2.0
+
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -24,6 +27,39 @@ function getExtraFeatures() {
     return features;
 }
 
+function getModelFeatures(model, artifactName) {
+  if(model.definitions && model.definitions[artifactName] && model.definitions[artifactName].features) {
+    return model.definitions[artifactName].features;
+  }
+  return undefined;
+}
+
+/*
+* Returns the feature from the stepContext or the model. If in both the feature is disabled then we return undefined.
+* */
+function getFeatureFromContext(stepContext, model, featureName) {
+  let modelFeature = undefined;
+  if (model) {
+    const modelFeatures = getModelFeatures(model, model.info.title);
+    if (modelFeatures && modelFeatures[featureName]) {
+      modelFeature = modelFeatures[featureName];
+    }
+  }
+
+  let stepFeature = undefined;
+  if (stepContext && stepContext.features && stepContext.features[featureName]) {
+    stepFeature = stepContext.features[featureName];
+  }
+  if (stepFeature && stepFeature.enabled) {
+    return stepFeature;
+  } else if (modelFeature && modelFeature.enabled) {
+    return modelFeature;
+  }
+  return undefined;
+}
+
 export default {
-    getExtraFeatures
+  getExtraFeatures,
+  getModelFeatures,
+  getFeatureFromContext
 };

marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/features/protected-collections.mjs

@@ -19,31 +19,32 @@ import core from "/data-hub/5/artifacts/core.mjs";
 import consts from "/data-hub/5/impl/consts.mjs";
 import hubUtils from "/data-hub/5/impl/hub-utils.mjs";
 const sec = require("/MarkLogic/security.xqy");
+import featuresUtils from "./features-util.mjs";
 
 const INFO_EVENT = consts.TRACE_CORE;
 const DEBUG_EVENT = consts.TRACE_CORE_DEBUG;
 const DEBUG_ENABLED = xdmp.traceEnabled(DEBUG_EVENT);
 
 function protectCollections(collections, permissions) {
-    collections.forEach(coll => {
-        try {
-            if (DEBUG_ENABLED) {
-                hubUtils.hubTrace(DEBUG_EVENT, `Protecting collection ${coll}`);
-            }
-            let permissionsSplit = permissions.split(",");
-            if (permissionsSplit.length % 2 !== 0) {
-                hubUtils.hubTrace(INFO_EVENT, `While adding protected collections, cannot protect collection ${coll}. Invalid permissions: ${permissions}`);
-            } else {
-                let permissionsSec = [];
-                for (let i = 0; i < permissionsSplit.length; i++) {
-                    permissionsSec.push(xdmp.permission(permissionsSplit[i], permissionsSplit[++i], "element"));
-                }
-                sec.protectCollection(coll, permissionsSec);
-            }
-        } catch(e) {
-            hubUtils.hubTrace(INFO_EVENT, `Could not protect collection ${coll}; Reason: ${e.message}`);
-        }
-    })
+  collections.forEach(coll => {
+      try {
+          if (DEBUG_ENABLED) {
+              hubUtils.hubTrace(DEBUG_EVENT, `Protecting collection ${coll}`);
+          }
+          let permissionsSplit = permissions.split(",");
+          if (permissionsSplit.length % 2 !== 0) {
+              hubUtils.hubTrace(INFO_EVENT, `While adding protected collections, cannot protect collection ${coll}. Invalid permissions: ${permissions}`);
+          } else {
+              let permissionsSec = [];
+              for (let i = 0; i < permissionsSplit.length; i++) {
+                  permissionsSec.push(xdmp.permission(permissionsSplit[i], permissionsSplit[++i], "element"));
+              }
+              sec.protectCollection(coll, permissionsSec);
+          }
+      } catch(e) {
+          hubUtils.hubTrace(INFO_EVENT, `Could not protect collection ${coll}; Reason: ${e.message}`);
+      }
+  })
 }
 
 function onArtifactPublish (artifactType, artifactName) {
@@ -70,76 +71,57 @@ function onInstanceSave(context, model, contentArray) {
       return contentArray;
   }
 
-  let flagAddCollectionsToObject = false;
-
   const modelName = model.info.title;
-  if(featureEnabled(model.definitions[modelName])) {
-      hubUtils.hubTrace(INFO_EVENT, `Processing protected collections feature for an instance of ${modelName}`);
-
-      const modelFeature = model.definitions[modelName].features["protectedCollections"];
-      if (modelFeature.permissions) {
-          permissions = modelFeature.permissions;
-      }
-      if (modelFeature.collections) {
-          collections = modelFeature.collections;
-      }
-      flagAddCollectionsToObject = true;
-  }
+  const feature = featuresUtils.getFeatureFromContext(stepContext, model, 'protectedCollections');
+  if(feature) {
+    hubUtils.hubTrace(INFO_EVENT, `Processing protected collections feature for an instance of ${modelName} while running ${stepContext.name}`);
 
-  if(featureEnabled(stepContext)) {
-    hubUtils.hubTrace(INFO_EVENT, `Processing protected collections feature for an instance while running ${stepContext.name}`);
-
-    const stepFeature = stepContext.features["protectedCollections"]
-    if (stepFeature.permissions) {
-      permissions = stepFeature.permissions;
+    if (feature.permissions) {
+        permissions = feature.permissions;
     }
-    if (stepFeature.collections) {
-      collections = stepFeature.collections;
+    if (feature.collections) {
+        collections = feature.collections;
     }
-    flagAddCollectionsToObject = true;
-  }
-
-  if (flagAddCollectionsToObject){
-      addCollectionsToObject(collections, permissions, contentArray);
+    addCollectionsToObject(collections, permissions, contentArray);
   }
 
   hubUtils.hubTrace(INFO_EVENT, `Finished processing protected collections feature for an instance of ${modelName} while running ${stepContext.name}`);
 
-  // Returning the contentArray for testing porpouses
+  // Returning the contentArray for testing purposes
   return contentArray;
 }
 
 function featureEnabled(artifact) {
-    if (artifact.features && artifact.features["protectedCollections"]) {
-        return artifact.features["protectedCollections"].enabled ? artifact.features["protectedCollections"].enabled : false
-    }
-    return false;
+  if (artifact.features && artifact.features["protectedCollections"]) {
+      return artifact.features["protectedCollections"].enabled ? artifact.features["protectedCollections"].enabled : false
+  }
+  return false;
 }
 
 function addCollectionsToObject(collections, permissions, contentArray) {
-    if(Array.isArray(collections) && collections.length > 0) {
-
-        // add collections to the contentObject if it doesn't have them
-        contentArray.forEach(contentObject => {
-            const contentCollections = contentObject.context.collections || [];
-            let newCollections = [];
-            if (Array.isArray(contentCollections)) {
-                collections.forEach(coll => {
-                    if (!contentObject.context.collections.includes(coll)) {
-                        newCollections.push(coll);
-                    }
-                })
-                contentObject.context.collections = contentCollections.concat(newCollections);
-            } else {
-                contentObject.context.collections = collections;
-            }
-        });
-        //confirm that the collections are protected
-        protectCollections(collections, permissions);
-    }
+  if(Array.isArray(collections) && collections.length > 0) {
+
+      // add collections to the contentObject if it doesn't have them
+      contentArray.forEach(contentObject => {
+          const contentCollections = contentObject.context.collections || [];
+          let newCollections = [];
+          if (Array.isArray(contentCollections)) {
+              collections.forEach(coll => {
+                  if (!contentObject.context.collections.includes(coll)) {
+                      newCollections.push(coll);
+                  }
+              })
+              contentObject.context.collections = contentCollections.concat(newCollections);
+          } else {
+              contentObject.context.collections = collections;
+          }
+      });
+      //confirm that the collections are protected
+      protectCollections(collections, permissions);
+  }
 }
 
 export default {
-    onArtifactPublish,
-    onInstanceSave
+  onArtifactPublish,
+  onInstanceSave
 };