diff --git a/openfn-69066751-5f2c-459c-b42e-feba1c802383-state.json b/openfn-69066751-5f2c-459c-b42e-feba1c802383-state.json index 3011934..4235ae6 100644 --- a/openfn-69066751-5f2c-459c-b42e-feba1c802383-state.json +++ b/openfn-69066751-5f2c-459c-b42e-feba1c802383-state.json @@ -4,6 +4,10 @@ "description": null, "inserted_at": "2024-10-30T06:18:14Z", "updated_at": "2024-11-12T08:57:07Z", + "scheduled_deletion": null, + "history_retention_period": 90, + "dataclip_retention_period": null, + "retention_policy": "retain_all", "project_credentials": { "mtuchi@openfn.org-mtuchi-github-token": { "id": "55508835-a9c3-4283-813e-ae6f9c4958fd", @@ -41,11 +45,6 @@ "owner": "mtuchi@openfn.org" } }, - "scheduled_deletion": null, - "history_retention_period": 90, - "dataclip_retention_period": null, - "retention_policy": "retain_all", - "requires_mfa": false, "workflows": { "wf-3-generate-optsmap": { "id": "c9d7d47a-5502-4f52-bc30-950746a8b473", @@ -169,11 +168,70 @@ } } }, + "fetch-metadata-and-generate-opts-json": { + "id": "182c38b7-fa91-4bc3-82ed-ebd92e26be8f", + "name": "fetch-metadata-and-generate-opts-json", + "inserted_at": "2024-11-12T16:06:06.024347Z", + "lock_version": 70, + "triggers": { + "webhook": { + "enabled": true, + "id": "654e0bc2-22ae-4033-9090-18760ab28ea2", + "type": "webhook" + } + }, + "jobs": { + "Get-metadata-file-from-Sharepoint": { + "id": "aab663a0-6bcc-42fa-9f53-146d8fa44061", + "name": "Get metadata file from Sharepoint", + "body": "fn(state => {\n state.sheets = ['OptionSets', 'identifiers'];\n state.siteId =\n 'openfnorg.sharepoint.com,4724a499-afbc-4ded-a371-34ae40bf5d8d,1d20a7d4-a6f1-407c-aa77-76bd47bb0f32';\n return state;\n});\n\ngetDrive(\n {\n id: $.siteId,\n owner: 'sites',\n },\n 'default'\n);\n\ngetFile('/msf-metadata/LIME EMR - Iraq Metadata - Release 1.xlsx', {\n metadata: true,\n});\n\nfn(state => {\n const itemId = state.data.id;\n const driveId = state.drives.default.id;\n state.workbookBase = `sites/${state.siteId}/drives/${driveId}/items/${itemId}/workbook`;\n return state;\n});\n\nget(\n `${$.workbookBase}/worksheets('omrs-form-metadata')/usedRange`,\n {},\n state => {\n const [headers, ...rows] = state.data.values;\n state.formMetadata = rows\n .map(row =>\n row.reduce((obj, value, index) => {\n if (value) {\n obj[headers[index]] = value;\n }\n return obj;\n }, {})\n )\n .filter(obj => Object.keys(obj).length > 0 && obj['Active']);\n\n state.sheets.push(\n ...state.formMetadata.map(obj => obj['OMRS form sheet name'])\n );\n return state;\n }\n);\n\neach(\n $.sheets,\n get(`${$.workbookBase}/worksheets('${$.data}')/usedRange`, {}, state => {\n const sheetName = state.references.at(-1);\n console.log('Fetched sheet: ', sheetName);\n state[sheetName] = state.data.values;\n return state;\n })\n);\n\nfn(state => {\n delete state.data;\n delete state.response;\n delete state.references;\n return state;\n});\n", + "adaptor": "@openfn/language-msgraph@latest", + "project_credential_id": "3276b5b6-25c8-4a64-b55d-cf02f82d21e3" + }, + "Map-metadata-file-to-option-set-Json-format": { + "id": "b5743b64-3250-4dbf-8d74-256a1c7e9623", + "name": "Map metadata file to option-set Json format", + "body": "const isValidValue = value => value !== '' && value !== 'NA';\n\nconst mapArrayToObject = (item, keys) => {\n return item.reduce((acc, value, idx) => {\n acc[keys[idx]] = value;\n return acc;\n }, {});\n};\n\nconst safeKeyValuePairs = arr => {\n if (arr === null || arr === undefined) {\n return arr;\n }\n const mappedArr = arr.slice(2).map(item => mapArrayToObject(item, arr[1]));\n try {\n return mappedArr\n .filter(\n o => isValidValue(o['External ID']) && isValidValue(o['DHIS2 DE UID'])\n )\n .reduce((acc, value) => {\n acc[value['DHIS2 DE UID']] = value['External ID'];\n return acc;\n }, {});\n } catch (error) {\n console.error(`Error processing ${arr}:`, error);\n return arr; // Return original value if processing fails\n }\n};\n\n//New OptionSet Mappping\nconst questionKeyValuePairs = arr => {\n if (arr === null || arr === undefined) {\n return arr;\n }\n const mappedArr = arr.slice(2).map(item => mapArrayToObject(item, arr[1]));\n try {\n return mappedArr.filter(\n o => isValidValue(o['External ID']) && isValidValue(o['DHIS2 Option Set UID'])\n )\n .map(value => ({\n [value['DHIS2 Option Set UID']]: value['External ID']\n }));\n } catch (error) {\n console.error(`Error processing ${arr}:`, error);\n return arr; // Return original value if processing fails\n }\n};\n\nfn(state => {\n const { OptionSets, identifiers } = state;\n const keys = OptionSets[1];\n\n state.optsMap = OptionSets.slice(2)\n .map(item => mapArrayToObject(item, keys))\n .filter(\n o =>\n (isValidValue(o['External ID']) &&\n isValidValue(o['DHIS2 DE full name'])) ||\n (isValidValue(o['value.display - Answers']) &&\n isValidValue(o['DHIS2 Option code']))\n )\n .map(o => {\n const optionSetUid = o['DHIS2 Option Set UID']=='NA' ? '' : `-${o['DHIS2 Option Set UID']}`; \n //then build an answerKeyUid = DEuid + OptionSetUid\n const answerKeyUid = `${o['External ID']}${optionSetUid}`; \n\n return {\n 'DHIS2 Option Set UID': o['DHIS2 Option Set UID'],\n 'DHIS2 Option name': o['DHIS2 Option name'],\n 'DHIS2 Option UID': o['DHIS2 Option UID'],\n 'DHIS2 Option Code': o['DHIS2 Option code'],\n 'value.display - Answers': o['Answers'],\n 'value.uuid - External ID': o['External ID'],\n 'answerMappingUid': answerKeyUid,\n 'DHIS2 DE full name': o['DHIS2 DE full name'],\n 'DHIS2 DE UID': o['DHIS2 DE UID'],\n 'OptionSet name': o['OptionSet name'],\n 'DHIS2 Option Set name': o['DHIS2 Option Set name'],\n };\n });\n\n const [iheaders, ...irows] = identifiers;\n state.identifiers = irows\n .map(row =>\n row.reduce((obj, value, index) => {\n if (value != null && value !== '') {\n obj[iheaders[index]] = value;\n }\n return obj;\n }, {})\n )\n .filter(obj => Object.keys(obj).length > 0);\n return state;\n});\n\n\nfn(state => {\n const { formMetadata, optsMap, identifiers } = state;\n\n const formMaps = formMetadata.reduce((acc, form) => {\n const formName = form['OMRS form sheet name'];\n acc[form['OMRS form.uuid']] = {\n formName,\n orgUnit: form['DHIS2 orgUnit ID'],\n programId: form['DHIS2 program ID'],\n programStage: form['DHIS2 programStage ID'],\n dataValueMap: safeKeyValuePairs(state[formName]),\n optionSetMap: questionKeyValuePairs(state[formName]),\n };\n\n return acc;\n }, {});\n\n const optionSetKey = Object.entries(formMaps).reduce((acc, [formKey, formValue]) => {\n \n // Iterate over each object in the optionSetMap array\n formValue.optionSetMap.forEach(item => {\n // Extract the single key-value pair from each object in the array\n const [originalKey, originalValue] = Object.entries(item)[0];\n // Reverse key-value, adding form prefix\n acc[`${formKey}-${originalValue}`] = originalKey;\n });\n return acc;\n }, {});\n\n // const optionSetKey = Object.entries(formMaps).reduce((acc, [formKey, formValue]) => {\n // // Iterate over each optionSetMap entry and reverse key-value, adding form prefix\n // Object.entries(formValue.optionSetMap).forEach(([originalKey, originalValue]) => {\n // acc[`${formKey}-${originalValue}`] = originalKey;\n // });\n // return acc;\n // }, {})\n\n\n return { formMaps, formMetadata, optsMap, optionSetKey, identifiers };\n});\n", + "adaptor": "@openfn/language-common@latest", + "project_credential_id": null + }, + "Save-option-set-json-to-github": { + "id": "b224b966-35f4-43ea-90e9-1f5dfcb56416", + "name": "Save option-set json to github", + "body": "const metadataPath =\n 'repos/OpenFn/openfn-lime-pilot/contents/metadata/collections.json';\n\nget(metadataPath, {\n headers: {\n 'user-agent': 'OpenFn',\n },\n query: {\n ref: 'collections',\n },\n});\n\nfn(state => {\n const { formMaps, formMetadata, optsMap, data, identifiers, optionSetKey } =\n state;\n\n state.body = {\n message: 'Update metadata content',\n committer: {\n name: 'Emmanuel Evance',\n email: 'mtuchidev@gmail.com',\n },\n content: util.encode(\n JSON.stringify({\n optionSetKey,\n optsMap,\n formMaps,\n identifiers,\n formMetadata,\n })\n ),\n sha: data.sha,\n branch: 'collections',\n };\n\n return state;\n});\n\nput(\n metadataPath,\n {\n body: $.body,\n headers: {\n Accept: 'application/vnd.github+json',\n 'X-GitHub-Api-Version': '2022-11-28',\n 'user-agent': 'OpenFn',\n },\n },\n ({ body, data, references, response, ...state }) => state\n);\n", + "adaptor": "@openfn/language-http@latest", + "project_credential_id": "55508835-a9c3-4283-813e-ae6f9c4958fd" + } + }, + "edges": { + "Get-metadata-file-from-Sharepoint->Map-metadata-file-to-option-set-Json-format": { + "enabled": true, + "id": "4ec86a4d-17b0-447c-a816-42f1d95d7f25", + "source_job_id": "aab663a0-6bcc-42fa-9f53-146d8fa44061", + "condition_type": "on_job_success", + "target_job_id": "b5743b64-3250-4dbf-8d74-256a1c7e9623" + }, + "Map-metadata-file-to-option-set-Json-format->Save-option-set-json-to-github": { + "enabled": true, + "id": "820d203a-1e9c-494d-8fba-c7c962209de7", + "source_job_id": "b5743b64-3250-4dbf-8d74-256a1c7e9623", + "condition_type": "on_job_success", + "target_job_id": "b224b966-35f4-43ea-90e9-1f5dfcb56416" + }, + "webhook->Get-metadata-file-from-Sharepoint": { + "enabled": true, + "id": "441b595b-441a-4938-be44-be49c32f5cc4", + "source_trigger_id": "654e0bc2-22ae-4033-9090-18760ab28ea2", + "condition_type": "always", + "target_job_id": "aab663a0-6bcc-42fa-9f53-146d8fa44061" + } + } + }, "wf2-omrs-dhis2": { "id": "c886121c-0a91-45b5-ae30-7d0922b693be", "name": "wf2-omrs-dhis2", - "inserted_at": "2024-11-12T15:41:47.478143Z", - "lock_version": 103, + "inserted_at": "2024-11-12T16:16:40.298307Z", + "lock_version": 104, "triggers": { "cron": { "enabled": false, @@ -207,7 +265,7 @@ "Get-TEIs-and-Map-Answers": { "id": "793f4fe1-d717-4caa-a617-2b5e2b19880d", "name": "Get TEIs and Map Answers", - "body": "const delay = ms => new Promise(resolve => setTimeout(resolve, ms));\n\neach(\n $.encounters,\n get(\n 'tracker/trackedEntities',\n {\n orgUnit: $.orgUnit,\n program: $.program,\n filter: [`AYbfTPYMNJH:Eq:${$.data.patient.uuid}`],\n fields: '*',\n },\n {},\n async state => {\n const encounter = state.references.at(-1);\n console.log(encounter.patient.uuid, 'Encounter patient uuid');\n\n const { trackedEntity, enrollments } = state.data?.instances?.[0] || {};\n if (trackedEntity && enrollments) {\n state.TEIs ??= {};\n state.TEIs[encounter.patient.uuid] = {\n trackedEntity,\n enrollments,\n enrollment: enrollments[0]?.enrollment,\n };\n }\n\n await delay(2000);\n return state;\n }\n )\n);\n\nconst processAnswer = (\n answer,\n conceptUuid,\n dataElement,\n optsMap,\n optionSetKey\n) => {\n // console.log('Has answer', conceptUuid, dataElement);\n return typeof answer.value === 'object'\n ? processObjectAnswer(\n answer,\n conceptUuid,\n dataElement,\n optsMap,\n optionSetKey\n )\n : processOtherAnswer(answer, conceptUuid, dataElement);\n};\n\nconst processObjectAnswer = (\n answer,\n conceptUuid,\n dataElement,\n optsMap,\n optionSetKey\n) => {\n if (isDiagnosisByPsychologist(conceptUuid, dataElement)) {\n return '' + answer.value.uuid === '278401ee-3d6f-4c65-9455-f1c16d0a7a98';\n }\n return findMatchingOption(answer, optsMap, optionSetKey);\n};\n\nconst processOtherAnswer = (answer, conceptUuid, dataElement) => {\n if (isPhq9Score(answer.value, conceptUuid, dataElement)) {\n return getRangePhq(answer.value);\n }\n return answer.value;\n};\n\nconst processNoAnswer = (data, conceptUuid, dataElement) => {\n // console.log('No answer', conceptUuid, dataElement);\n if (isEncounterDate(conceptUuid, dataElement)) {\n return data.encounterDatetime.replace('+0000', '');\n }\n return '';\n};\n\nconst findMatchingOption = (answer, optsMap, optionSetKey) => {\n const optionKey = `${answer.formUuid}-${answer.concept.uuid}`; \n\n //const matchingOptionSet = optionSetKey[answer.concept.uuid];\n const matchingOptionSet = optionSetKey[optionKey];\n console.log('optionKey', optionKey);\n console.log('conceptUid', answer.concept.uuid);\n console.log('value uid', answer.value.uuid);\n console.log('value', answer.value.display);\n console.log('matchingOptionSet', matchingOptionSet);\n\n //const answerKey = answerMappingUid\n\n const matchingOption = optsMap.find(\n o =>\n o['value.uuid - External ID'] === answer.value.uuid &&\n o['DHIS2 Option Set UID'] === matchingOptionSet\n )?.['DHIS2 Option Code'] || answer.value.display; //TODO: revisit this logic if optionSet not found\n\n console.log('matchingOption value', matchingOption)\n\n // //TBD if we want to keep thse --> TODO: Revisit this logic!\n // if (matchingOption?.toLowerCase() === 'no') {\n // console.log('FALSE option', matchingOption)\n // return 'FALSE';\n // }\n // if (matchingOption?.toLowerCase() === 'yes') {\n // console.log('TRUE option', matchingOption)\n // return 'TRUE';\n // }\n //=========================================//\n\n return matchingOption || '';\n};\n\nconst isEncounterDate = (conceptUuid, dataElement) => {\n return (\n conceptUuid === 'encounter-date' &&\n ['CXS4qAJH2qD', 'I7phgLmRWQq', 'yUT7HyjWurN'].includes(dataElement)\n );\n};\n\nconst isDiagnosisByPsychologist = (conceptUuid, dataElement) =>\n conceptUuid === '722dd83a-c1cf-48ad-ac99-45ac131ccc96' &&\n dataElement === 'pN4iQH4AEzk';\n\nconst isPhq9Score = (value, conceptUuid, dataElement) =>\n typeof value === 'number' &&\n conceptUuid === '5f3d618e-5c89-43bd-8c79-07e4e98c2f23' &&\n dataElement === 'tsFOVnlc6lz';\n\nconst getRangePhq = input => {\n if (input >= 20) return '>20';\n if (input >= 15) return '15_19';\n if (input >= 10) return '10_14';\n if (input >= 5) return '5_9';\n return '0_4';\n};\n\nconst dataValuesMapping = (data, dataValueMap, optsMap, optionSetKey) => {\n return Object.keys(dataValueMap)\n .map(dataElement => {\n const conceptUuid = dataValueMap[dataElement];\n const obsAnswer = data.obs.find(o => o.concept.uuid === conceptUuid);\n const answer = {\n ...obsAnswer,\n formUuid: data.form.uuid\n };\n const value = answer\n ? processAnswer(answer, conceptUuid, dataElement, optsMap, optionSetKey)\n : processNoAnswer(data, conceptUuid, dataElement);\n\n return { dataElement, value };\n })\n .filter(d => d);\n};\n\n// Prepare DHIS2 data model for create events\nfn(state => {\n const createEvent = (data, state) => {\n const { trackedEntity, enrollment } = state.TEIs[data.patient.uuid] || {};\n\n if (!trackedEntity || !enrollment) {\n return null;\n }\n\n return {\n program: state.program,\n orgUnit: state.orgUnit,\n trackedEntity,\n enrollment,\n occurredAt: data.encounterDatetime.replace('+0000', ''),\n };\n };\n\n const handleMissingRecord = (data, state) => {\n const { uuid, display } = data.patient;\n\n console.log(uuid, 'Patient is missing trackedEntity && enrollment');\n\n state.missingRecords ??= {};\n state.missingRecords[uuid] ??= {\n encounters: [],\n patient: display,\n };\n\n state.missingRecords[uuid].encounters.push(data);\n };\n\n const processEncounter = (data, state) => {\n const event = createEvent(data, state);\n if (!event) {\n handleMissingRecord(data, state);\n return null;\n }\n\n const form = state.formMaps[data.form.uuid];\n if (!form?.dataValueMap) {\n return null;\n }\n\n return {\n ...event,\n programStage: form.programStage,\n dataValues: dataValuesMapping(\n data,\n form.dataValueMap,\n state.optsMap,\n state.optionSetKey\n ),\n };\n };\n\n state.encountersMapping = state.encounters\n .map(data => processEncounter(data, state))\n .filter(Boolean);\n\n return state;\n});\n// const findMatchingOption = (answer, optsMap, optionSetKey) => {\n// const answerKeyUid = optionSetKey[answer.concept.uuid];\n\n// const matchingOption = optsMap.find(\n// (o) => o[\"DHIS2 answerKeyUid\"] === answerKeyUid\n// )?.[\"DHIS2 Option Code\"];\n\n// //TBD if we want this.. TODO: revisit this logic\n// if (matchingOption === \"no\") {\n// return \"FALSE\";\n// }\n// if (matchingOption === \"yes\") {\n// return \"TRUE\";\n// }\n// //======//\n// return matchingOption || \"\";\n// };\n\n//=== Original logic modified on Nov 11 =========//\n// const findMatchingOption = (answer, optsMap) => {\n// const matchingOption = optsMap.find(\n// o => o['value.uuid - External ID'] === answer.value.uuid\n// )?.['DHIS2 Option Code'];\n\n// if (matchingOption === 'no') {\n// return 'FALSE';\n// }\n// if (matchingOption === 'yes') {\n// return 'TRUE';\n// }\n// return matchingOption || '';\n// };\n", + "body": "const delay = ms => new Promise(resolve => setTimeout(resolve, ms));\n\neach(\n $.encounters,\n get(\n 'tracker/trackedEntities',\n {\n orgUnit: $.orgUnit,\n program: $.program,\n filter: [`AYbfTPYMNJH:Eq:${$.data.patient.uuid}`],\n fields: '*',\n },\n {},\n async state => {\n const encounter = state.references.at(-1);\n console.log(encounter.patient.uuid, 'Encounter patient uuid');\n\n const { trackedEntity, enrollments } = state.data?.instances?.[0] || {};\n if (trackedEntity && enrollments) {\n state.TEIs ??= {};\n state.TEIs[encounter.patient.uuid] = {\n trackedEntity,\n enrollments,\n enrollment: enrollments[0]?.enrollment,\n };\n }\n\n await delay(2000);\n return state;\n }\n )\n);\n\nconst processAnswer = (\n answer,\n conceptUuid,\n dataElement,\n optsMap,\n optionSetKey\n) => {\n // console.log('Has answer', conceptUuid, dataElement);\n return typeof answer.value === 'object'\n ? processObjectAnswer(\n answer,\n conceptUuid,\n dataElement,\n optsMap,\n optionSetKey\n )\n : processOtherAnswer(answer, conceptUuid, dataElement);\n};\n\nconst processObjectAnswer = (\n answer,\n conceptUuid,\n dataElement,\n optsMap,\n optionSetKey\n) => {\n if (isDiagnosisByPsychologist(conceptUuid, dataElement)) {\n return '' + answer.value.uuid === '278401ee-3d6f-4c65-9455-f1c16d0a7a98';\n }\n return findMatchingOption(answer, optsMap, optionSetKey);\n};\n\nconst processOtherAnswer = (answer, conceptUuid, dataElement) => {\n if (isPhq9Score(answer.value, conceptUuid, dataElement)) {\n return getRangePhq(answer.value);\n }\n return answer.value;\n};\n\nconst processNoAnswer = (data, conceptUuid, dataElement) => {\n // console.log('No answer', conceptUuid, dataElement);\n if (isEncounterDate(conceptUuid, dataElement)) {\n return data.encounterDatetime.replace('+0000', '');\n }\n return '';\n};\n\nconst findMatchingOption = (answer, optsMap, optionSetKey) => {\n const optionKey = `${answer.formUuid}-${answer.concept.uuid}`; \n\n //const matchingOptionSet = optionSetKey[answer.concept.uuid];\n const matchingOptionSet = optionSetKey[optionKey];\n console.log('optionKey', optionKey);\n console.log('conceptUid', answer.concept.uuid);\n console.log('value uid', answer.value.uuid);\n console.log('value', answer.value.display);\n console.log('matchingOptionSet', matchingOptionSet);\n\n //const answerKey = answerMappingUid\n\n const matchingOption = optsMap.find(\n o =>\n o['value.uuid - External ID'] === answer.value.uuid &&\n o['DHIS2 Option Set UID'] === matchingOptionSet\n )?.['DHIS2 Option Code'] || answer.value.display; //TODO: revisit this logic if optionSet not found\n\n console.log('matchingOption value', matchingOption)\n\n // to convert ALL caps to lowercase per DHIS2 api\n if (matchingOption === 'FALSE') {\n console.log('false option', matchingOption)\n return 'false';\n }\n if (matchingOption === 'TRUE') {\n console.log('true option', matchingOption)\n return 'true';\n }\n ////=========================================//\n\n return matchingOption || '';\n};\n\nconst isEncounterDate = (conceptUuid, dataElement) => {\n return (\n conceptUuid === 'encounter-date' &&\n ['CXS4qAJH2qD', 'I7phgLmRWQq', 'yUT7HyjWurN'].includes(dataElement)\n );\n};\n\nconst isDiagnosisByPsychologist = (conceptUuid, dataElement) =>\n conceptUuid === '722dd83a-c1cf-48ad-ac99-45ac131ccc96' &&\n dataElement === 'pN4iQH4AEzk';\n\nconst isPhq9Score = (value, conceptUuid, dataElement) =>\n typeof value === 'number' &&\n conceptUuid === '5f3d618e-5c89-43bd-8c79-07e4e98c2f23' &&\n dataElement === 'tsFOVnlc6lz';\n\nconst getRangePhq = input => {\n if (input >= 20) return '>20';\n if (input >= 15) return '15_19';\n if (input >= 10) return '10_14';\n if (input >= 5) return '5_9';\n return '0_4';\n};\n\nconst dataValuesMapping = (data, dataValueMap, optsMap, optionSetKey) => {\n return Object.keys(dataValueMap)\n .map(dataElement => {\n const conceptUuid = dataValueMap[dataElement];\n const obsAnswer = data.obs.find(o => o.concept.uuid === conceptUuid);\n const answer = {\n ...obsAnswer,\n formUuid: data.form.uuid\n };\n const value = answer\n ? processAnswer(answer, conceptUuid, dataElement, optsMap, optionSetKey)\n : processNoAnswer(data, conceptUuid, dataElement);\n\n return { dataElement, value };\n })\n .filter(d => d);\n};\n\n// Prepare DHIS2 data model for create events\nfn(state => {\n const createEvent = (data, state) => {\n const { trackedEntity, enrollment } = state.TEIs[data.patient.uuid] || {};\n\n if (!trackedEntity || !enrollment) {\n return null;\n }\n\n return {\n program: state.program,\n orgUnit: state.orgUnit,\n trackedEntity,\n enrollment,\n occurredAt: data.encounterDatetime.replace('+0000', ''),\n };\n };\n\n const handleMissingRecord = (data, state) => {\n const { uuid, display } = data.patient;\n\n console.log(uuid, 'Patient is missing trackedEntity && enrollment');\n\n state.missingRecords ??= {};\n state.missingRecords[uuid] ??= {\n encounters: [],\n patient: display,\n };\n\n state.missingRecords[uuid].encounters.push(data);\n };\n\n const processEncounter = (data, state) => {\n const event = createEvent(data, state);\n if (!event) {\n handleMissingRecord(data, state);\n return null;\n }\n\n const form = state.formMaps[data.form.uuid];\n if (!form?.dataValueMap) {\n return null;\n }\n\n return {\n ...event,\n programStage: form.programStage,\n dataValues: dataValuesMapping(\n data,\n form.dataValueMap,\n state.optsMap,\n state.optionSetKey\n ),\n };\n };\n\n state.encountersMapping = state.encounters\n .map(data => processEncounter(data, state))\n .filter(Boolean);\n\n return state;\n});\n// const findMatchingOption = (answer, optsMap, optionSetKey) => {\n// const answerKeyUid = optionSetKey[answer.concept.uuid];\n\n// const matchingOption = optsMap.find(\n// (o) => o[\"DHIS2 answerKeyUid\"] === answerKeyUid\n// )?.[\"DHIS2 Option Code\"];\n\n// //TBD if we want this.. TODO: revisit this logic\n// if (matchingOption === \"no\") {\n// return \"FALSE\";\n// }\n// if (matchingOption === \"yes\") {\n// return \"TRUE\";\n// }\n// //======//\n// return matchingOption || \"\";\n// };\n\n//=== Original logic modified on Nov 11 =========//\n// const findMatchingOption = (answer, optsMap) => {\n// const matchingOption = optsMap.find(\n// o => o['value.uuid - External ID'] === answer.value.uuid\n// )?.['DHIS2 Option Code'];\n\n// if (matchingOption === 'no') {\n// return 'FALSE';\n// }\n// if (matchingOption === 'yes') {\n// return 'TRUE';\n// }\n// return matchingOption || '';\n// };\n", "adaptor": "@openfn/language-dhis2@5.0.1", "project_credential_id": "9716980f-1b35-4295-adde-b05bc47002da" }, @@ -292,65 +350,7 @@ "target_job_id": "793f4fe1-d717-4caa-a617-2b5e2b19880d" } } - }, - "fetch-metadata-and-generate-opts-json": { - "id": "182c38b7-fa91-4bc3-82ed-ebd92e26be8f", - "name": "fetch-metadata-and-generate-opts-json", - "inserted_at": "2024-11-12T16:06:06.024347Z", - "lock_version": 70, - "triggers": { - "webhook": { - "enabled": true, - "id": "654e0bc2-22ae-4033-9090-18760ab28ea2", - "type": "webhook" - } - }, - "jobs": { - "Get-metadata-file-from-Sharepoint": { - "id": "aab663a0-6bcc-42fa-9f53-146d8fa44061", - "name": "Get metadata file from Sharepoint", - "body": "fn(state => {\n state.sheets = ['OptionSets', 'identifiers'];\n state.siteId =\n 'openfnorg.sharepoint.com,4724a499-afbc-4ded-a371-34ae40bf5d8d,1d20a7d4-a6f1-407c-aa77-76bd47bb0f32';\n return state;\n});\n\ngetDrive(\n {\n id: $.siteId,\n owner: 'sites',\n },\n 'default'\n);\n\ngetFile('/msf-metadata/LIME EMR - Iraq Metadata - Release 1.xlsx', {\n metadata: true,\n});\n\nfn(state => {\n const itemId = state.data.id;\n const driveId = state.drives.default.id;\n state.workbookBase = `sites/${state.siteId}/drives/${driveId}/items/${itemId}/workbook`;\n return state;\n});\n\nget(\n `${$.workbookBase}/worksheets('omrs-form-metadata')/usedRange`,\n {},\n state => {\n const [headers, ...rows] = state.data.values;\n state.formMetadata = rows\n .map(row =>\n row.reduce((obj, value, index) => {\n if (value) {\n obj[headers[index]] = value;\n }\n return obj;\n }, {})\n )\n .filter(obj => Object.keys(obj).length > 0 && obj['Active']);\n\n state.sheets.push(\n ...state.formMetadata.map(obj => obj['OMRS form sheet name'])\n );\n return state;\n }\n);\n\neach(\n $.sheets,\n get(`${$.workbookBase}/worksheets('${$.data}')/usedRange`, {}, state => {\n const sheetName = state.references.at(-1);\n console.log('Fetched sheet: ', sheetName);\n state[sheetName] = state.data.values;\n return state;\n })\n);\n\nfn(state => {\n delete state.data;\n delete state.response;\n delete state.references;\n return state;\n});\n", - "adaptor": "@openfn/language-msgraph@latest", - "project_credential_id": "3276b5b6-25c8-4a64-b55d-cf02f82d21e3" - }, - "Map-metadata-file-to-option-set-Json-format": { - "id": "b5743b64-3250-4dbf-8d74-256a1c7e9623", - "name": "Map metadata file to option-set Json format", - "body": "const isValidValue = value => value !== '' && value !== 'NA';\n\nconst mapArrayToObject = (item, keys) => {\n return item.reduce((acc, value, idx) => {\n acc[keys[idx]] = value;\n return acc;\n }, {});\n};\n\nconst safeKeyValuePairs = arr => {\n if (arr === null || arr === undefined) {\n return arr;\n }\n const mappedArr = arr.slice(2).map(item => mapArrayToObject(item, arr[1]));\n try {\n return mappedArr\n .filter(\n o => isValidValue(o['External ID']) && isValidValue(o['DHIS2 DE UID'])\n )\n .reduce((acc, value) => {\n acc[value['DHIS2 DE UID']] = value['External ID'];\n return acc;\n }, {});\n } catch (error) {\n console.error(`Error processing ${arr}:`, error);\n return arr; // Return original value if processing fails\n }\n};\n\n//New OptionSet Mappping\nconst questionKeyValuePairs = arr => {\n if (arr === null || arr === undefined) {\n return arr;\n }\n const mappedArr = arr.slice(2).map(item => mapArrayToObject(item, arr[1]));\n try {\n return mappedArr.filter(\n o => isValidValue(o['External ID']) && isValidValue(o['DHIS2 Option Set UID'])\n )\n .map(value => ({\n [value['DHIS2 Option Set UID']]: value['External ID']\n }));\n } catch (error) {\n console.error(`Error processing ${arr}:`, error);\n return arr; // Return original value if processing fails\n }\n};\n\nfn(state => {\n const { OptionSets, identifiers } = state;\n const keys = OptionSets[1];\n\n state.optsMap = OptionSets.slice(2)\n .map(item => mapArrayToObject(item, keys))\n .filter(\n o =>\n (isValidValue(o['External ID']) &&\n isValidValue(o['DHIS2 DE full name'])) ||\n (isValidValue(o['value.display - Answers']) &&\n isValidValue(o['DHIS2 Option code']))\n )\n .map(o => {\n const optionSetUid = o['DHIS2 Option Set UID']=='NA' ? '' : `-${o['DHIS2 Option Set UID']}`; \n //then build an answerKeyUid = DEuid + OptionSetUid\n const answerKeyUid = `${o['External ID']}${optionSetUid}`; \n\n return {\n 'DHIS2 Option Set UID': o['DHIS2 Option Set UID'],\n 'DHIS2 Option name': o['DHIS2 Option name'],\n 'DHIS2 Option UID': o['DHIS2 Option UID'],\n 'DHIS2 Option Code': o['DHIS2 Option code'],\n 'value.display - Answers': o['Answers'],\n 'value.uuid - External ID': o['External ID'],\n 'answerMappingUid': answerKeyUid,\n 'DHIS2 DE full name': o['DHIS2 DE full name'],\n 'DHIS2 DE UID': o['DHIS2 DE UID'],\n 'OptionSet name': o['OptionSet name'],\n 'DHIS2 Option Set name': o['DHIS2 Option Set name'],\n };\n });\n\n const [iheaders, ...irows] = identifiers;\n state.identifiers = irows\n .map(row =>\n row.reduce((obj, value, index) => {\n if (value != null && value !== '') {\n obj[iheaders[index]] = value;\n }\n return obj;\n }, {})\n )\n .filter(obj => Object.keys(obj).length > 0);\n return state;\n});\n\n\nfn(state => {\n const { formMetadata, optsMap, identifiers } = state;\n\n const formMaps = formMetadata.reduce((acc, form) => {\n const formName = form['OMRS form sheet name'];\n acc[form['OMRS form.uuid']] = {\n formName,\n orgUnit: form['DHIS2 orgUnit ID'],\n programId: form['DHIS2 program ID'],\n programStage: form['DHIS2 programStage ID'],\n dataValueMap: safeKeyValuePairs(state[formName]),\n optionSetMap: questionKeyValuePairs(state[formName]),\n };\n\n return acc;\n }, {});\n\n const optionSetKey = Object.entries(formMaps).reduce((acc, [formKey, formValue]) => {\n \n // Iterate over each object in the optionSetMap array\n formValue.optionSetMap.forEach(item => {\n // Extract the single key-value pair from each object in the array\n const [originalKey, originalValue] = Object.entries(item)[0];\n // Reverse key-value, adding form prefix\n acc[`${formKey}-${originalValue}`] = originalKey;\n });\n return acc;\n }, {});\n\n // const optionSetKey = Object.entries(formMaps).reduce((acc, [formKey, formValue]) => {\n // // Iterate over each optionSetMap entry and reverse key-value, adding form prefix\n // Object.entries(formValue.optionSetMap).forEach(([originalKey, originalValue]) => {\n // acc[`${formKey}-${originalValue}`] = originalKey;\n // });\n // return acc;\n // }, {})\n\n\n return { formMaps, formMetadata, optsMap, optionSetKey, identifiers };\n});\n", - "adaptor": "@openfn/language-common@latest", - "project_credential_id": null - }, - "Save-option-set-json-to-github": { - "id": "b224b966-35f4-43ea-90e9-1f5dfcb56416", - "name": "Save option-set json to github", - "body": "const metadataPath =\n 'repos/OpenFn/openfn-lime-pilot/contents/metadata/collections.json';\n\nget(metadataPath, {\n headers: {\n 'user-agent': 'OpenFn',\n },\n query: {\n ref: 'collections',\n },\n});\n\nfn(state => {\n const { formMaps, formMetadata, optsMap, data, identifiers, optionSetKey } =\n state;\n\n state.body = {\n message: 'Update metadata content',\n committer: {\n name: 'Emmanuel Evance',\n email: 'mtuchidev@gmail.com',\n },\n content: util.encode(\n JSON.stringify({\n optionSetKey,\n optsMap,\n formMaps,\n identifiers,\n formMetadata,\n })\n ),\n sha: data.sha,\n branch: 'collections',\n };\n\n return state;\n});\n\nput(\n metadataPath,\n {\n body: $.body,\n headers: {\n Accept: 'application/vnd.github+json',\n 'X-GitHub-Api-Version': '2022-11-28',\n 'user-agent': 'OpenFn',\n },\n },\n ({ body, data, references, response, ...state }) => state\n);\n", - "adaptor": "@openfn/language-http@latest", - "project_credential_id": "55508835-a9c3-4283-813e-ae6f9c4958fd" - } - }, - "edges": { - "Get-metadata-file-from-Sharepoint->Map-metadata-file-to-option-set-Json-format": { - "enabled": true, - "id": "4ec86a4d-17b0-447c-a816-42f1d95d7f25", - "source_job_id": "aab663a0-6bcc-42fa-9f53-146d8fa44061", - "condition_type": "on_job_success", - "target_job_id": "b5743b64-3250-4dbf-8d74-256a1c7e9623" - }, - "Map-metadata-file-to-option-set-Json-format->Save-option-set-json-to-github": { - "enabled": true, - "id": "820d203a-1e9c-494d-8fba-c7c962209de7", - "source_job_id": "b5743b64-3250-4dbf-8d74-256a1c7e9623", - "condition_type": "on_job_success", - "target_job_id": "b224b966-35f4-43ea-90e9-1f5dfcb56416" - }, - "webhook->Get-metadata-file-from-Sharepoint": { - "enabled": true, - "id": "441b595b-441a-4938-be44-be49c32f5cc4", - "source_trigger_id": "654e0bc2-22ae-4033-9090-18760ab28ea2", - "condition_type": "always", - "target_job_id": "aab663a0-6bcc-42fa-9f53-146d8fa44061" - } - } } - } + }, + "requires_mfa": false } \ No newline at end of file diff --git a/workflows/wf2/5-get-teis.js b/workflows/wf2/5-get-teis.js index 06f473b..95a0d08 100644 --- a/workflows/wf2/5-get-teis.js +++ b/workflows/wf2/5-get-teis.js @@ -99,16 +99,16 @@ const findMatchingOption = (answer, optsMap, optionSetKey) => { console.log('matchingOption value', matchingOption) - // //TBD if we want to keep thse --> TODO: Revisit this logic! - // if (matchingOption?.toLowerCase() === 'no') { - // console.log('FALSE option', matchingOption) - // return 'FALSE'; - // } - // if (matchingOption?.toLowerCase() === 'yes') { - // console.log('TRUE option', matchingOption) - // return 'TRUE'; - // } - //=========================================// + // to convert ALL caps to lowercase per DHIS2 api + if (matchingOption === 'FALSE') { + console.log('false option', matchingOption) + return 'false'; + } + if (matchingOption === 'TRUE') { + console.log('true option', matchingOption) + return 'true'; + } + ////=========================================// return matchingOption || ''; };