Autogenerate optionSets and f01MphssBaseline (#49)

* use file path

* add wf3

* fetch data from sharepoint

* clean up fetch metadata

* Fetch excel file from shareipoint

* fix typo

* fix mapping

* Fetch more sheet from sharepoint (#54)

* fetch and map metadata

* remove commented code

openfn-cd92dd57-9a3c-4318-bdcb-f57a386cf811-spec.yaml

@@ -13,46 +13,60 @@ credentials:
   [email protected]:
     name: OpenMRS Demo
     owner: [email protected]
-  "[email protected](HTTP)":
+  '[email protected](HTTP)':
     name: 'OpenMRS Demo (HTTP)'
     owner: [email protected]
-  "[email protected]([email protected])-2":
+  '[email protected]([email protected])-2':
     name: 'googlesheets ([email protected])-2'
     owner: [email protected]
   [email protected]:
     name: mtuchi-github-token
     owner: [email protected]
 workflows:
-  fetch-metadata-and-generate-opts-json:
-    name: fetch-metadata-and-generate-opts-json
+  Sharepoint-test:
+    name: Sharepoint test
     jobs:
-      Get-metadata-file-from-Sharepoint:
-        name: Get metadata file from Sharepoint
+      New-job:
+        name: New job
         adaptor: '@openfn/language-msgraph@latest'
         credential: null
         body: |
-          
+
           // Check out the Job Writing Guide for help getting started:
           // https://docs.openfn.org/documentation/jobs/job-writing-guide
 
+    triggers:
+      webhook:
+        type: webhook
+        enabled: true
+    edges:
+      webhook->New-job:
+        source_trigger: webhook
+        target_job: New-job
+        condition_type: always
+        enabled: true
+  fetch-metadata-and-generate-opts-json:
+    name: fetch-metadata-and-generate-opts-json
+    jobs:
+      Get-metadata-file-from-Sharepoint:
+        name: Get metadata file from Sharepoint
+        adaptor: '@openfn/language-msgraph@latest'
+        credential: null
+        body:
+          path: ./workflows/wf3/1-fetch-metadata.json
       Map-metadata-file-to-option-set-Json-format:
         name: Map metadata file to option-set Json format
         adaptor: '@openfn/language-common@latest'
         credential: null
-        body: |
-          
-          // Check out the Job Writing Guide for help getting started:
-          // https://docs.openfn.org/documentation/jobs/job-writing-guide
+        body:
+          path: ./workflows/wf3/2-map-metadata.json
 
       Save-option-set-json-to-github:
         name: Save option-set json to github
         adaptor: '@openfn/language-http@latest'
         credential: null
-        body: |
-          
-          // Check out the Job Writing Guide for help getting started:
-          // https://docs.openfn.org/documentation/jobs/job-writing-guide
-
+        body:
+          path: ./workflows/wf3/3-save-options.json
     triggers:
       webhook:
         type: webhook
@@ -125,7 +139,7 @@ workflows:
         adaptor: '@openfn/language-msgraph@latest'
         credential: [email protected]
         body: |
-          
+
           // Check out the Job Writing Guide for help getting started:
           // https://docs.openfn.org/documentation/jobs/job-writing-guide
           get('sites/root/lists')
@@ -148,7 +162,7 @@ workflows:
         adaptor: '@openfn/language-googlesheets@latest'
         credential: '[email protected]([email protected])-2'
         body: |
-          
+
           getValues('1OuR7laA7Oc2QnoiT8S3Thhf-HNh7uFY8ILLEu-idXuk', 
           'optionsets_oct1')
 

workflows/wf3/1-fetch-metadata.js

@@ -0,0 +1,50 @@
+const toCamelCase = text => {
+  return text
+    .toLowerCase()
+    .replace(/[^a-zA-Z0-9]+(.)/g, (match, chr) => chr.toUpperCase());
+};
+
+const sheets = [
+  'OptionSets',
+  'F01-MHPSS Baseline',
+  'F02-MHPSS Follow-up',
+  'F03-mhGAP Baseline',
+  'F04-mhGAP Follow-up',
+  'F05-MHPSS Closure',
+];
+
+getDrive(
+  {
+    id: $.siteId,
+    owner: 'sites',
+  },
+  'default'
+);
+
+getFile('/msf-metadata/LIME EMR Iraq Metadata Oct 1.xlsx', {
+  metadata: true,
+});
+
+fn(state => {
+  const itemId = state.data.id;
+  const driveId = state.drives.default.id;
+  state.workbookBase = `sites/${state.siteId}/drives/${driveId}/items/${itemId}/workbook`;
+  return state;
+});
+
+each(
+  sheets,
+  get(`${$.workbookBase}/worksheets('${$.data}')/usedRange`, {}, state => {
+    const sheetName = toCamelCase(state.references.at(-1));
+    console.log('Fetched sheet: ', sheetName);
+    state[sheetName] = state.data.values;
+    return state;
+  })
+);
+
+fn(state => {
+  delete state.data;
+  delete state.response;
+  delete state.references;
+  return state;
+});

workflows/wf3/2-map-metadata.js

@@ -0,0 +1,81 @@
+const isValidValue = value => value !== '' && value !== 'NA';
+
+const mapArrayToObject = (item, keys) => {
+  return item.reduce((acc, value, idx) => {
+    acc[keys[idx]] = value;
+    return acc;
+  }, {});
+};
+fn(state => {
+  const { optionsets } = state;
+  const keys = optionsets[1];
+
+  const optsMap = optionsets.slice(2).map(item => mapArrayToObject(item, keys));
+
+  state.optionSets = optsMap
+    .filter(
+      o =>
+        isValidValue(o['External ID']) && isValidValue(o['DHIS2 DE full name'])
+    )
+    .map(o => {
+      return {
+        'value.display - Answers': o['Answers'],
+        'value.uuid - External ID': o['External ID'],
+        'DHIS2 DE full name': o['DHIS2 DE full name'],
+        'DHIS2 DE UID': o['DHIS2 DE UID'],
+        'OptionSet name': o['OptionSet name'],
+        'DHIS2 Option Set UID': o['DHIS2 Option Set name'],
+        'DHIS2 Option name': o['DHIS2 Option name'],
+        'DHIS2 Option UID': o['DHIS2 Option UID'],
+        'DHIS2 Option Code': o['DHIS2 Option code'],
+      };
+    });
+
+  return state;
+});
+
+const safeKeyValuePairs = arr => {
+  if (arr === null || arr === undefined) {
+    return arr;
+  }
+  const mappedArr = arr.slice(2).map(item => mapArrayToObject(item, arr[1]));
+  try {
+    return mappedArr
+      .filter(
+        o => isValidValue(o['External ID']) && isValidValue(o['DHIS2 DE UID'])
+      )
+      .reduce((acc, value) => {
+        acc[value['DHIS2 DE UID']] = value['External ID'];
+        return acc;
+      }, {});
+  } catch (error) {
+    console.error(`Error processing ${arr}:`, error);
+    return arr; // Return original value if processing fails
+  }
+};
+
+fn(
+  ({
+    optionSets,
+    f01MhpssBaseline,
+    f02MhpssFollowUp,
+    f03MhgapBaseline,
+    f04MhgapFollowUp,
+    f05MhpssClosure,
+  }) => {
+    const processedState = Object.fromEntries(
+      Object.entries({
+        f01MhpssBaseline,
+        f02MhpssFollowUp,
+        f03MhgapBaseline,
+        f04MhgapFollowUp,
+        f05MhpssClosure,
+      }).map(([key, value]) => [key, safeKeyValuePairs(value)])
+    );
+
+    return {
+      optionSets,
+      ...processedState,
+    };
+  }
+);

workflows/wf3/3-save-options.js

@@ -0,0 +1,50 @@
+const metadataPath =
+  'repos/OpenFn/openfn-lime-pilot/contents/metadata/metadata_mapping.json';
+
+get(metadataPath, {
+  headers: {
+    'user-agent': 'OpenFn',
+  },
+});
+
+fn(state => {
+  const {
+    optionSets,
+    f01MhpssBaseline,
+    f02MhpssFollowUp,
+    f03MhgapBaseline,
+    f04MhgapFollowUp,
+    f05MhpssClosure,
+    data,
+  } = state;
+
+  state.body = {
+    message: 'Update metadata content',
+    committer: {
+      name: 'Emmanuel Evance',
+      email: '[email protected]',
+    },
+    content: util.encode(
+      JSON.stringify({
+        optionSets,
+        f01MhpssBaseline,
+        f02MhpssFollowUp,
+        f03MhgapBaseline,
+        f04MhgapFollowUp,
+        f05MhpssClosure,
+      })
+    ),
+    sha: data.sha,
+  };
+
+  return state;
+});
+
+put(metadataPath, {
+  body: $.body,
+  headers: {
+    Accept: 'application/vnd.github+json',
+    'X-GitHub-Api-Version': '2022-11-28',
+    'user-agent': 'OpenFn',
+  },
+});

workflows/wf3/workflow.json

@@ -0,0 +1,32 @@
+{
+    "workflow": {
+        "steps": [
+            {
+                "id": "fetch-metadata",
+                "adaptor": "msgraph",
+                "state": {
+                    "siteId": "openfnorg.sharepoint.com,4724a499-afbc-4ded-a371-34ae40bf5d8d,1d20a7d4-a6f1-407c-aa77-76bd47bb0f32"
+                },
+                "configuration": "../tmp/msgraph-creds.json",
+                "expression": "1-fetch-metadata.js",
+                "next": {
+                    "map-metadata": "!state.errors"
+                }
+            },
+            {
+                "id": "map-metadata",
+                "adaptor": "common",
+                "expression": "2-map-metadata.js",
+                "next": {
+                    "save-options": "!state.errors"
+                }
+            },
+            {
+                "id": "save-options",
+                "adaptor": "http",
+                "expression": "3-save-options.js",
+                "configuration": "../tmp/github-creds.json"
+            }
+        ]
+    }
+}