diff --git a/openfn-cd92dd57-9a3c-4318-bdcb-f57a386cf811-spec.yaml b/openfn-cd92dd57-9a3c-4318-bdcb-f57a386cf811-spec.yaml index 1b4ade1..3d84586 100644 --- a/openfn-cd92dd57-9a3c-4318-bdcb-f57a386cf811-spec.yaml +++ b/openfn-cd92dd57-9a3c-4318-bdcb-f57a386cf811-spec.yaml @@ -13,46 +13,60 @@ credentials: mtuchi@openfn.org-OpenMRS-Demo: name: OpenMRS Demo owner: mtuchi@openfn.org - "mtuchi@openfn.org-OpenMRS-Demo-(HTTP)": + 'mtuchi@openfn.org-OpenMRS-Demo-(HTTP)': name: 'OpenMRS Demo (HTTP)' owner: mtuchi@openfn.org - "mtuchi@openfn.org-googlesheets-(mtuchi@openfn.org)-2": + 'mtuchi@openfn.org-googlesheets-(mtuchi@openfn.org)-2': name: 'googlesheets (mtuchi@openfn.org)-2' owner: mtuchi@openfn.org mtuchi@openfn.org-mtuchi-github-token: name: mtuchi-github-token owner: mtuchi@openfn.org workflows: - fetch-metadata-and-generate-opts-json: - name: fetch-metadata-and-generate-opts-json + Sharepoint-test: + name: Sharepoint test jobs: - Get-metadata-file-from-Sharepoint: - name: Get metadata file from Sharepoint + New-job: + name: New job adaptor: '@openfn/language-msgraph@latest' credential: null body: | - + // Check out the Job Writing Guide for help getting started: // https://docs.openfn.org/documentation/jobs/job-writing-guide + triggers: + webhook: + type: webhook + enabled: true + edges: + webhook->New-job: + source_trigger: webhook + target_job: New-job + condition_type: always + enabled: true + fetch-metadata-and-generate-opts-json: + name: fetch-metadata-and-generate-opts-json + jobs: + Get-metadata-file-from-Sharepoint: + name: Get metadata file from Sharepoint + adaptor: '@openfn/language-msgraph@latest' + credential: null + body: + path: ./workflows/wf3/1-fetch-metadata.json Map-metadata-file-to-option-set-Json-format: name: Map metadata file to option-set Json format adaptor: '@openfn/language-common@latest' credential: null - body: | - - // Check out the Job Writing Guide for help getting started: - // https://docs.openfn.org/documentation/jobs/job-writing-guide + body: + path: ./workflows/wf3/2-map-metadata.json Save-option-set-json-to-github: name: Save option-set json to github adaptor: '@openfn/language-http@latest' credential: null - body: | - - // Check out the Job Writing Guide for help getting started: - // https://docs.openfn.org/documentation/jobs/job-writing-guide - + body: + path: ./workflows/wf3/3-save-options.json triggers: webhook: type: webhook @@ -125,7 +139,7 @@ workflows: adaptor: '@openfn/language-msgraph@latest' credential: aleksa@openfn.org-AK-Sharepoint body: | - + // Check out the Job Writing Guide for help getting started: // https://docs.openfn.org/documentation/jobs/job-writing-guide get('sites/root/lists') @@ -148,7 +162,7 @@ workflows: adaptor: '@openfn/language-googlesheets@latest' credential: 'mtuchi@openfn.org-googlesheets-(mtuchi@openfn.org)-2' body: | - + getValues('1OuR7laA7Oc2QnoiT8S3Thhf-HNh7uFY8ILLEu-idXuk', 'optionsets_oct1') diff --git a/workflows/wf3/1-fetch-metadata.js b/workflows/wf3/1-fetch-metadata.js new file mode 100644 index 0000000..032a0d3 --- /dev/null +++ b/workflows/wf3/1-fetch-metadata.js @@ -0,0 +1,50 @@ +const toCamelCase = text => { + return text + .toLowerCase() + .replace(/[^a-zA-Z0-9]+(.)/g, (match, chr) => chr.toUpperCase()); +}; + +const sheets = [ + 'OptionSets', + 'F01-MHPSS Baseline', + 'F02-MHPSS Follow-up', + 'F03-mhGAP Baseline', + 'F04-mhGAP Follow-up', + 'F05-MHPSS Closure', +]; + +getDrive( + { + id: $.siteId, + owner: 'sites', + }, + 'default' +); + +getFile('/msf-metadata/LIME EMR Iraq Metadata Oct 1.xlsx', { + metadata: true, +}); + +fn(state => { + const itemId = state.data.id; + const driveId = state.drives.default.id; + state.workbookBase = `sites/${state.siteId}/drives/${driveId}/items/${itemId}/workbook`; + return state; +}); + +each( + sheets, + get(`${$.workbookBase}/worksheets('${$.data}')/usedRange`, {}, state => { + const sheetName = toCamelCase(state.references.at(-1)); + console.log('Fetched sheet: ', sheetName); + state[sheetName] = state.data.values; + return state; + }) +); + +fn(state => { + delete state.data; + delete state.response; + delete state.references; + return state; +}); diff --git a/workflows/wf3/2-map-metadata.js b/workflows/wf3/2-map-metadata.js new file mode 100644 index 0000000..583dd60 --- /dev/null +++ b/workflows/wf3/2-map-metadata.js @@ -0,0 +1,81 @@ +const isValidValue = value => value !== '' && value !== 'NA'; + +const mapArrayToObject = (item, keys) => { + return item.reduce((acc, value, idx) => { + acc[keys[idx]] = value; + return acc; + }, {}); +}; +fn(state => { + const { optionsets } = state; + const keys = optionsets[1]; + + const optsMap = optionsets.slice(2).map(item => mapArrayToObject(item, keys)); + + state.optionSets = optsMap + .filter( + o => + isValidValue(o['External ID']) && isValidValue(o['DHIS2 DE full name']) + ) + .map(o => { + return { + 'value.display - Answers': o['Answers'], + 'value.uuid - External ID': o['External ID'], + 'DHIS2 DE full name': o['DHIS2 DE full name'], + 'DHIS2 DE UID': o['DHIS2 DE UID'], + 'OptionSet name': o['OptionSet name'], + 'DHIS2 Option Set UID': o['DHIS2 Option Set name'], + 'DHIS2 Option name': o['DHIS2 Option name'], + 'DHIS2 Option UID': o['DHIS2 Option UID'], + 'DHIS2 Option Code': o['DHIS2 Option code'], + }; + }); + + return state; +}); + +const safeKeyValuePairs = arr => { + if (arr === null || arr === undefined) { + return arr; + } + const mappedArr = arr.slice(2).map(item => mapArrayToObject(item, arr[1])); + try { + return mappedArr + .filter( + o => isValidValue(o['External ID']) && isValidValue(o['DHIS2 DE UID']) + ) + .reduce((acc, value) => { + acc[value['DHIS2 DE UID']] = value['External ID']; + return acc; + }, {}); + } catch (error) { + console.error(`Error processing ${arr}:`, error); + return arr; // Return original value if processing fails + } +}; + +fn( + ({ + optionSets, + f01MhpssBaseline, + f02MhpssFollowUp, + f03MhgapBaseline, + f04MhgapFollowUp, + f05MhpssClosure, + }) => { + const processedState = Object.fromEntries( + Object.entries({ + f01MhpssBaseline, + f02MhpssFollowUp, + f03MhgapBaseline, + f04MhgapFollowUp, + f05MhpssClosure, + }).map(([key, value]) => [key, safeKeyValuePairs(value)]) + ); + + return { + optionSets, + ...processedState, + }; + } +); diff --git a/workflows/wf3/3-save-options.js b/workflows/wf3/3-save-options.js new file mode 100644 index 0000000..304e504 --- /dev/null +++ b/workflows/wf3/3-save-options.js @@ -0,0 +1,50 @@ +const metadataPath = + 'repos/OpenFn/openfn-lime-pilot/contents/metadata/metadata_mapping.json'; + +get(metadataPath, { + headers: { + 'user-agent': 'OpenFn', + }, +}); + +fn(state => { + const { + optionSets, + f01MhpssBaseline, + f02MhpssFollowUp, + f03MhgapBaseline, + f04MhgapFollowUp, + f05MhpssClosure, + data, + } = state; + + state.body = { + message: 'Update metadata content', + committer: { + name: 'Emmanuel Evance', + email: 'mtuchidev@gmail.com', + }, + content: util.encode( + JSON.stringify({ + optionSets, + f01MhpssBaseline, + f02MhpssFollowUp, + f03MhgapBaseline, + f04MhgapFollowUp, + f05MhpssClosure, + }) + ), + sha: data.sha, + }; + + return state; +}); + +put(metadataPath, { + body: $.body, + headers: { + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'user-agent': 'OpenFn', + }, +}); diff --git a/workflows/wf3/workflow.json b/workflows/wf3/workflow.json new file mode 100644 index 0000000..6509fe7 --- /dev/null +++ b/workflows/wf3/workflow.json @@ -0,0 +1,32 @@ +{ + "workflow": { + "steps": [ + { + "id": "fetch-metadata", + "adaptor": "msgraph", + "state": { + "siteId": "openfnorg.sharepoint.com,4724a499-afbc-4ded-a371-34ae40bf5d8d,1d20a7d4-a6f1-407c-aa77-76bd47bb0f32" + }, + "configuration": "../tmp/msgraph-creds.json", + "expression": "1-fetch-metadata.js", + "next": { + "map-metadata": "!state.errors" + } + }, + { + "id": "map-metadata", + "adaptor": "common", + "expression": "2-map-metadata.js", + "next": { + "save-options": "!state.errors" + } + }, + { + "id": "save-options", + "adaptor": "http", + "expression": "3-save-options.js", + "configuration": "../tmp/github-creds.json" + } + ] + } +} \ No newline at end of file