diff --git a/README.md b/README.md index 289188c9..54e4cbe3 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,6 @@ be either persisted to a file system or to a running HL7 FHIR endpoint. toFHIR consists of the following modules: - `tofhir-engine`: The core module of toFHIR which includes the main functionality of the tool. - `tofhir-server`: A standalone web server module that provides a REST API to run mapping jobs via `tohir-engine` and manage the mapping job definitions. -- `tofhir-log-server`: A standalone web server module that provides a REST API to query the logs of the mapping job execution results. - `tofhir-server-common`: Holds common files like server configuration or errors for server implementations. - `tofhir-common`: Contains model and utility classes shared across various modules. - `tofhir-rxnorm`: Provides a client implementation to access the RxNorm API and a FHIR Path Function library to utilize its API functionalities in mapping definitions. @@ -272,11 +271,6 @@ webserver = { password = null } } - -# The service from where tofhir-server will read the logs. -log-service = { - endpoint = "http://localhost:8086/tofhir-logs" -} ``` After the server is up and running, the engine will be available via the REST API. @@ -1221,3 +1215,32 @@ Or both. This will delete the source files after processing/mapping and save the While `archiveMode` works on a file-based basis, `saveErroneousRecords` works for each record/row in the source data. Please also note that, the `archiveMode` config is only applicable for the file system source type. + +## Mapping Job Results with EFK Stack +This project utilizes the EFK Stack (Elasticsearch, Fluentd, and Kibana) to visualize the results of mapping job executions. +Once the EFK Stack is started using the provided [docker-compose.yml](docker/docker-compose.yml), Kibana can be accessed at http://localhost:5601. +### Initializing Kibana +Follow these steps to initialize Kibana with predefined configurations, dashboards, and indexes: +#### Importing Visualizations, Dashboards, and Indexes +1. Navigate to http://localhost:5601/app/management/kibana/objects +2. Click on the `Import` button and import [export.ndjson](docker/kibana-data/export.ndjson) + +![Importing Objects](readme-assets%2Fkibana-saved-objects.png) +#### Creating an Index Template for 'ignore_above' Properties of Strings +To handle potentially long log messages, increase the default value of `ignore_above` for string properties. + +1. Go to http://localhost:5601/app/dev_tools#/console +2. Copy and paste the contents of [index_template.txt](docker/kibana-data/index_template.txt) into the console and execute the request. + +![Index Template API Response](readme-assets%2Fkibana-index-template.png) + +### Kibana Dashboards +After running mapping jobs, you can view their logs via Kibana dashboards. +### Executions Dashboard +This dashboard provides an overview of each execution's result. +![Executions Dashboard](readme-assets%2Fkibana-executions-dashboard.png) +To analyze a specific execution, hover over the execution ID and click on the plus icon next to it. Then, select the `Go to Dashboard` option as shown below: +![Go to Dashboard](readme-assets%2Fkibana-go-to-dashboard.png) +### Execution Details Dashboard +This dashboard displays the results of individual mapping tasks and any corresponding errors. +![Execution Details Dashboard](readme-assets%2Fkibana-execution-details.png) \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5f594442..09bdd841 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,21 +1,10 @@ version: '3.9' services: - tofhir-log-server: - image: srdc/tofhir-log-server:latest - container_name: tofhir-log-server - hostname: tofhir-log-server - ports: - - "8086:8086" - networks: - - tofhir-network - volumes: - - './tofhir-docker-logs:/usr/local/tofhir/logs' tofhir-server: image: srdc/tofhir-server:latest container_name: tofhir-server hostname: tofhir-server environment: - - LOG_SERVICE_ENDPOINT=http://tofhir-log-server:8086/tofhir-logs - FHIR_DEFINITIONS_ENDPOINT=http://onfhir:8080/fhir - FHIR_REPO_URL=http://onfhir:8080/fhir - APP_CONF_FILE=/usr/local/tofhir/conf/docker/tofhir-server.conf @@ -34,7 +23,44 @@ services: - "8087:80" networks: - tofhir-network + # Elasticsearch, Fluentd and Kibana stack for mapping job log management + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.15.2 + container_name: elasticsearch + environment: + - discovery.type=single-node + ports: + - "9200:9200" + networks: + - tofhir-network + volumes: + - elasticsearch_data:/usr/share/elasticsearch/data + fluentd: + build: ./fluentd + container_name: fluentd + volumes: + - ./fluentd/conf:/fluentd/etc + ports: + - "24224:24224" + - "24224:24224/udp" + networks: + - tofhir-network + # TODO: Investigate if it's possible to populate predefined Kibana visualizations, dashboards, and indexes + # using Docker. This could involve making API calls or utilizing volumes. + # The current approach is the utilize Kibana UI as described in the README file. + kibana: + image: docker.elastic.co/kibana/kibana:7.15.2 + container_name: kibana + ports: + - "5601:5601" + networks: + - tofhir-network + depends_on: + - elasticsearch networks: tofhir-network: name: onfhir-network external: true +volumes: + elasticsearch_data: + driver: local \ No newline at end of file diff --git a/docker/fluentd/Dockerfile b/docker/fluentd/Dockerfile new file mode 100644 index 00000000..eb33162b --- /dev/null +++ b/docker/fluentd/Dockerfile @@ -0,0 +1,15 @@ +#Specifies the base image for building the Fluentd Docker image +FROM fluent/fluentd:v1.11-1 +#Switches to the root user account to perform package installations and configurations. +USER root +#Installs necessary packages and dependencies using the Alpine package manager (apk). +RUN apk add --no-cache --update --virtual .build-deps \ + sudo build-base ruby-dev \ + && sudo gem install faraday -v 2.8.1 \ + && sudo gem install faraday-net_http -v 3.0.2 \ + && sudo gem install fluent-plugin-elasticsearch \ + && sudo gem sources --clear-all \ + && apk del .build-deps \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +# Switches back to the fluent user account to run Fluentd. +USER fluent \ No newline at end of file diff --git a/docker/fluentd/conf/fluent.conf b/docker/fluentd/conf/fluent.conf new file mode 100644 index 00000000..34a7fe61 --- /dev/null +++ b/docker/fluentd/conf/fluent.conf @@ -0,0 +1,23 @@ +// Defines a Fluentd configuration for forwarding logs to Elasticsearch. + + + // Specifies the input plugin type as 'forward', which receives log entries forwarded by Fluentd clients. + @type forward + // Specifies the port on which the forward input plugin listens for incoming log entries. + port 24224 + + + + // Specifies the output plugin type as 'elasticsearch', which sends log entries to an Elasticsearch cluster. + @type elasticsearch + // Specifies the host address of the Elasticsearch cluster. + host elasticsearch + // Specifies the port number of the Elasticsearch cluster. + port 9200 + // Configures the output format to be compatible with Logstash. + logstash_format true + // Specifies the prefix to be added to the index name in Elasticsearch. + logstash_prefix fluentd + // Specifies the interval at which buffered log entries are flushed to Elasticsearch. + flush_interval 5s + \ No newline at end of file diff --git a/docker/kibana-data/export.ndjson b/docker/kibana-data/export.ndjson new file mode 100644 index 00000000..9d207978 --- /dev/null +++ b/docker/kibana-data/export.ndjson @@ -0,0 +1,14 @@ +{"attributes":{"accessCount":22,"accessDate":1714047836559,"createDate":1713268358357,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data)),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)&show-query-input=true&show-time-filter=true"},"coreMigrationVersion":"7.15.2","id":"0c52344e47af8f96bbb54b2c911cbdf7","references":[],"sort":[1714047836559,953],"type":"url","updated_at":"2024-04-25T12:23:56.559Z","version":"WzIwNjcwLDEzXQ=="} +{"attributes":{"fieldAttrs":"{\"result\":{\"count\":2}}","fieldFormatMap":"{\"@timestamp-original\":{\"id\":\"string\",\"params\":{\"parsedUrl\":{\"origin\":\"http://localhost:5601\",\"pathname\":\"/app/management/data/index_management/indices\",\"basePath\":\"\"},\"pattern\":\"0,0.[000]\"}}}","fields":"[{\"count\":0,\"script\":\"doc['@timestamp']\",\"lang\":\"painless\",\"name\":\"@timestamp-original\",\"type\":\"string\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"script\":\"if(doc['numOfNotMapped'].size() == 0){\\n return 0;\\n}\\nelse if(doc['numOfNotMapped'].value == -1){\\n return 0;\\n}\\nelse {\\n return doc['numOfNotMapped'].value;\\n}\",\"lang\":\"painless\",\"name\":\"normalizedNumOfNotMapped\",\"type\":\"number\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false,\"customLabel\":\"\"},{\"count\":0,\"script\":\"if(doc['numOfFailedWrites'].size() == 0){\\r\\n return 0;\\r\\n}\\r\\nelse if(doc['numOfFailedWrites'].value == -1){\\r\\n return 0;\\r\\n}\\r\\nelse {\\r\\n return doc['numOfFailedWrites'].value;\\r\\n}\",\"lang\":\"painless\",\"name\":\"normalizedNumOfFailedWrites\",\"type\":\"number\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false,\"customLabel\":\"\"},{\"count\":0,\"script\":\"if(doc['numOfInvalids'].size() == 0){\\r\\n return 0;\\r\\n}\\r\\nelse if(doc['numOfInvalids'].value == -1){\\r\\n return 0;\\r\\n}\\r\\nelse {\\r\\n return doc['numOfInvalids'].value;\\r\\n}\",\"lang\":\"painless\",\"name\":\"normalizedNumOfInvalids\",\"type\":\"number\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false,\"customLabel\":\"\"},{\"count\":0,\"script\":\"if(doc['numOfFhirResources'].size() == 0){\\r\\n return 0;\\r\\n}\\r\\nelse if(doc['numOfFhirResources'].value == -1){\\r\\n return 0;\\r\\n}\\r\\nelse {\\r\\n return doc['numOfFhirResources'].value;\\r\\n}\",\"lang\":\"painless\",\"name\":\"normalizedNumOfFhirResources\",\"type\":\"number\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false,\"customLabel\":\"\"}]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"fluentd*","typeMeta":"{}"},"coreMigrationVersion":"7.15.2","id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1713962901850,795],"type":"index-pattern","updated_at":"2024-04-24T12:48:21.850Z","version":"WzIwNTIwLDEzXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.15.2\",\"type\":\"visualization\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":3,\"i\":\"ecd350a2-0317-4f66-b2cb-80596e4a0689\"},\"panelIndex\":\"ecd350a2-0317-4f66-b2cb-80596e4a0689\",\"embeddableConfig\":{\"savedVis\":{\"title\":\"\",\"description\":\"\",\"type\":\"markdown\",\"params\":{\"fontSize\":12,\"openLinksInNewTab\":false,\"markdown\":\"Go back to [Executions Dashboard](http://localhost:5601/goto/2a517919d8081cb04ccd2c61a0e1c74a)\\n\"},\"uiState\":{},\"data\":{\"aggs\":[],\"searchSource\":{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}}},\"enhancements\":{}}},{\"version\":\"7.15.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":3,\"w\":48,\"h\":14,\"i\":\"9f083d19-1515-40d5-b078-918dd8b0f5ed\"},\"panelIndex\":\"9f083d19-1515-40d5-b078-918dd8b0f5ed\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsDatatable\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-layer-045875b2-5639-4e09-a9f0-06a2bc94627c\"}],\"state\":{\"visualization\":{\"layerId\":\"045875b2-5639-4e09-a9f0-06a2bc94627c\",\"layerType\":\"data\",\"columns\":[{\"isTransposed\":false,\"columnId\":\"a9105d68-f41f-4a6a-bcf2-88afb2753911\",\"width\":448.83333333333337},{\"isTransposed\":false,\"columnId\":\"200d4609-5416-4b35-9eb7-bbf58e4e940b\",\"hidden\":true},{\"isTransposed\":false,\"columnId\":\"e71b0e35-d05f-47c0-bac4-80e0c501d6cc\"},{\"isTransposed\":false,\"columnId\":\"5d44faff-382c-484d-9fa8-d4660f26805e\"},{\"isTransposed\":false,\"columnId\":\"bda664e6-f527-4e55-880a-be71839aef3a\"},{\"isTransposed\":false,\"columnId\":\"08108aa9-c895-48be-b8c6-5888ef415407\"},{\"columnId\":\"f431fd4e-b333-4ec2-a4b6-2cebf08b179c\",\"isTransposed\":false}],\"sorting\":{\"columnId\":\"a9105d68-f41f-4a6a-bcf2-88afb2753911\",\"direction\":\"asc\"}},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"045875b2-5639-4e09-a9f0-06a2bc94627c\":{\"columns\":{\"a9105d68-f41f-4a6a-bcf2-88afb2753911\":{\"label\":\"Mapping URL\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"mappingUrl.keyword\",\"isBucketed\":true,\"params\":{\"size\":1000,\"orderBy\":{\"type\":\"column\",\"columnId\":\"200d4609-5416-4b35-9eb7-bbf58e4e940b\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false},\"customLabel\":true},\"200d4609-5416-4b35-9eb7-bbf58e4e940b\":{\"label\":\"Count of records\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\"},\"e71b0e35-d05f-47c0-bac4-80e0c501d6cc\":{\"label\":\"Num Of Invalids\",\"dataType\":\"number\",\"operationType\":\"sum\",\"sourceField\":\"normalizedNumOfInvalids\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true},\"5d44faff-382c-484d-9fa8-d4660f26805e\":{\"label\":\"Num Of Not Mapped\",\"dataType\":\"number\",\"operationType\":\"sum\",\"sourceField\":\"normalizedNumOfNotMapped\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true},\"bda664e6-f527-4e55-880a-be71839aef3a\":{\"label\":\"Num Of Failed Writes\",\"dataType\":\"number\",\"operationType\":\"sum\",\"sourceField\":\"normalizedNumOfFailedWrites\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true},\"08108aa9-c895-48be-b8c6-5888ef415407\":{\"label\":\"Num Of Fhir Resources\",\"dataType\":\"number\",\"operationType\":\"sum\",\"sourceField\":\"normalizedNumOfFhirResources\",\"isBucketed\":false,\"scale\":\"ratio\",\"customLabel\":true},\"f431fd4e-b333-4ec2-a4b6-2cebf08b179c\":{\"label\":\"Result\",\"dataType\":\"string\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ordinal\",\"sourceField\":\"result.keyword\",\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true}},\"columnOrder\":[\"a9105d68-f41f-4a6a-bcf2-88afb2753911\",\"200d4609-5416-4b35-9eb7-bbf58e4e940b\",\"f431fd4e-b333-4ec2-a4b6-2cebf08b179c\",\"e71b0e35-d05f-47c0-bac4-80e0c501d6cc\",\"5d44faff-382c-484d-9fa8-d4660f26805e\",\"bda664e6-f527-4e55-880a-be71839aef3a\",\"08108aa9-c895-48be-b8c6-5888ef415407\"],\"incompleteColumns\":{}}}}}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Mapping Tasks\"},{\"version\":\"7.15.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":17,\"w\":48,\"h\":15,\"i\":\"cc5f4b91-a8a1-4a46-b6af-fcf27bd2cc9a\"},\"panelIndex\":\"cc5f4b91-a8a1-4a46-b6af-fcf27bd2cc9a\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsDatatable\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-layer-da995f15-8b40-4a27-8e2f-ecf572761658\"}],\"state\":{\"visualization\":{\"layerId\":\"da995f15-8b40-4a27-8e2f-ecf572761658\",\"layerType\":\"data\",\"columns\":[{\"isTransposed\":false,\"columnId\":\"f53828c9-3afc-4907-bb6a-2c535ccf27d3\"},{\"isTransposed\":false,\"columnId\":\"12bdb40b-542b-40e4-a4f6-0eb2e460d0ac\"},{\"isTransposed\":false,\"columnId\":\"eef089a3-5333-4e64-9a67-5801a4c8ca9b\",\"hidden\":true}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"da995f15-8b40-4a27-8e2f-ecf572761658\":{\"columns\":{\"f53828c9-3afc-4907-bb6a-2c535ccf27d3\":{\"label\":\"Stack Trace\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"throwable.keyword\",\"isBucketed\":true,\"params\":{\"size\":1000,\"orderBy\":{\"type\":\"column\",\"columnId\":\"eef089a3-5333-4e64-9a67-5801a4c8ca9b\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false},\"customLabel\":true},\"12bdb40b-542b-40e4-a4f6-0eb2e460d0ac\":{\"label\":\"Mapping URL\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"mappingUrl.keyword\",\"isBucketed\":true,\"params\":{\"size\":1000,\"orderBy\":{\"type\":\"column\",\"columnId\":\"eef089a3-5333-4e64-9a67-5801a4c8ca9b\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false},\"customLabel\":true},\"eef089a3-5333-4e64-9a67-5801a4c8ca9b\":{\"label\":\"Count of records\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\"}},\"columnOrder\":[\"12bdb40b-542b-40e4-a4f6-0eb2e460d0ac\",\"f53828c9-3afc-4907-bb6a-2c535ccf27d3\",\"eef089a3-5333-4e64-9a67-5801a4c8ca9b\"],\"incompleteColumns\":{}}}}}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"Mapping Task Execution Errors\"},{\"version\":\"7.15.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":32,\"w\":48,\"h\":11,\"i\":\"8f8ec50c-37e3-4c3f-aa05-889b981892d2\"},\"panelIndex\":\"8f8ec50c-37e3-4c3f-aa05-889b981892d2\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsDatatable\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-layer-531e2a4d-3863-4033-8c76-2a1386ab1e17\"}],\"state\":{\"visualization\":{\"columns\":[{\"isTransposed\":false,\"columnId\":\"e2b16ad6-47e2-473f-bddb-66d67a486b98\"},{\"isTransposed\":false,\"columnId\":\"d4664133-34fb-4fbb-ae84-5ad4b43fdf7e\"},{\"isTransposed\":false,\"columnId\":\"3905b6ac-675a-4033-817e-ab5c677d7391\"},{\"isTransposed\":false,\"columnId\":\"09621465-1768-488c-95e1-95a3aa63f289\"},{\"isTransposed\":false,\"columnId\":\"6283eb86-046f-4e45-8503-dba05d1934a8\"},{\"isTransposed\":false,\"columnId\":\"18410b40-aa47-4442-a4e3-ec8fa015c107\",\"hidden\":false}],\"layerId\":\"531e2a4d-3863-4033-8c76-2a1386ab1e17\",\"layerType\":\"data\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"531e2a4d-3863-4033-8c76-2a1386ab1e17\":{\"columns\":{\"e2b16ad6-47e2-473f-bddb-66d67a486b98\":{\"label\":\"Mapping URL\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"mappingUrl.keyword\",\"isBucketed\":true,\"params\":{\"size\":1000,\"orderBy\":{\"type\":\"column\",\"columnId\":\"18410b40-aa47-4442-a4e3-ec8fa015c107\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false},\"customLabel\":true},\"d4664133-34fb-4fbb-ae84-5ad4b43fdf7e\":{\"label\":\"Code\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"errorCode.keyword\",\"isBucketed\":true,\"params\":{\"size\":100,\"orderBy\":{\"type\":\"column\",\"columnId\":\"18410b40-aa47-4442-a4e3-ec8fa015c107\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false},\"customLabel\":true},\"3905b6ac-675a-4033-817e-ab5c677d7391\":{\"label\":\"Description\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"errorDesc.keyword\",\"isBucketed\":true,\"params\":{\"size\":1000,\"orderBy\":{\"type\":\"column\",\"columnId\":\"18410b40-aa47-4442-a4e3-ec8fa015c107\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false},\"customLabel\":true},\"09621465-1768-488c-95e1-95a3aa63f289\":{\"label\":\"Expression\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"errorExpr.keyword\",\"isBucketed\":true,\"params\":{\"size\":1000,\"orderBy\":{\"type\":\"column\",\"columnId\":\"18410b40-aa47-4442-a4e3-ec8fa015c107\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":true},\"customLabel\":true},\"6283eb86-046f-4e45-8503-dba05d1934a8\":{\"label\":\"Source\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"source.keyword\",\"isBucketed\":true,\"params\":{\"size\":1000,\"orderBy\":{\"type\":\"column\",\"columnId\":\"18410b40-aa47-4442-a4e3-ec8fa015c107\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false},\"customLabel\":true},\"18410b40-aa47-4442-a4e3-ec8fa015c107\":{\"label\":\"Count\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"customLabel\":true}},\"columnOrder\":[\"e2b16ad6-47e2-473f-bddb-66d67a486b98\",\"d4664133-34fb-4fbb-ae84-5ad4b43fdf7e\",\"3905b6ac-675a-4033-817e-ab5c677d7391\",\"09621465-1768-488c-95e1-95a3aa63f289\",\"6283eb86-046f-4e45-8503-dba05d1934a8\",\"18410b40-aa47-4442-a4e3-ec8fa015c107\"],\"incompleteColumns\":{}}}}}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Mapping Errors\"}]","timeRestore":false,"title":"Execution Details","version":1},"coreMigrationVersion":"7.15.2","id":"994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5","migrationVersion":{"dashboard":"7.15.0"},"references":[{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"9f083d19-1515-40d5-b078-918dd8b0f5ed:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"9f083d19-1515-40d5-b078-918dd8b0f5ed:indexpattern-datasource-layer-045875b2-5639-4e09-a9f0-06a2bc94627c","type":"index-pattern"},{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"cc5f4b91-a8a1-4a46-b6af-fcf27bd2cc9a:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"cc5f4b91-a8a1-4a46-b6af-fcf27bd2cc9a:indexpattern-datasource-layer-da995f15-8b40-4a27-8e2f-ecf572761658","type":"index-pattern"},{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"8f8ec50c-37e3-4c3f-aa05-889b981892d2:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"8f8ec50c-37e3-4c3f-aa05-889b981892d2:indexpattern-datasource-layer-531e2a4d-3863-4033-8c76-2a1386ab1e17","type":"index-pattern"}],"sort":[1714045615795,945],"type":"dashboard","updated_at":"2024-04-25T11:46:55.795Z","version":"WzIwNjYyLDEzXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.15.2\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":16,\"i\":\"d62f4d3d-027b-408d-b8ad-62404b90a057\"},\"panelIndex\":\"d62f4d3d-027b-408d-b8ad-62404b90a057\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsDatatable\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5\",\"name\":\"indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca\"},{\"name\":\"drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId\",\"type\":\"dashboard\",\"id\":\"994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5\"}],\"state\":{\"visualization\":{\"layerId\":\"a67e6635-b164-4c45-a563-bbe4ca458fca\",\"layerType\":\"data\",\"columns\":[{\"isTransposed\":false,\"columnId\":\"73954412-33dd-4460-9ed0-25fb11272df8\",\"width\":297.66666666666663},{\"columnId\":\"6c1cf201-6770-49ee-9056-9bc1f009f5da\",\"isTransposed\":false},{\"columnId\":\"567b3cde-4bc5-4155-82bb-8c0f1097af27\",\"isTransposed\":false},{\"columnId\":\"6e4b599c-1d09-46c1-83b7-d66df0337009\",\"isTransposed\":false},{\"columnId\":\"8c65e9ca-7985-43ec-ae2e-7791dff2e7d3\",\"isTransposed\":false,\"width\":213.66666666666666},{\"columnId\":\"76358e4d-51dc-4bb7-9763-45c422e51e5b\",\"isTransposed\":false}],\"sorting\":{\"columnId\":\"8c65e9ca-7985-43ec-ae2e-7791dff2e7d3\",\"direction\":\"desc\"}},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"a67e6635-b164-4c45-a563-bbe4ca458fca\":{\"columns\":{\"73954412-33dd-4460-9ed0-25fb11272df8\":{\"label\":\"Execution ID\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"executionId.keyword\",\"isBucketed\":true,\"params\":{\"size\":5,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":true},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false},\"customLabel\":true},\"6c1cf201-6770-49ee-9056-9bc1f009f5daX0\":{\"label\":\"Part of count(kql='result : SUCCESS')\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"result : SUCCESS\",\"language\":\"kuery\"},\"customLabel\":true},\"6c1cf201-6770-49ee-9056-9bc1f009f5da\":{\"label\":\"Success Count\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"count(kql='result : SUCCESS')\",\"isFormulaBroken\":false},\"references\":[\"6c1cf201-6770-49ee-9056-9bc1f009f5daX0\"],\"customLabel\":true},\"6e4b599c-1d09-46c1-83b7-d66df0337009X0\":{\"label\":\"Part of Partial Success Count\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"result : PARTIAL_SUCCESS\",\"language\":\"kuery\"},\"customLabel\":true},\"6e4b599c-1d09-46c1-83b7-d66df0337009\":{\"label\":\"Partial Success Count\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"count(kql='result : PARTIAL_SUCCESS')\",\"isFormulaBroken\":false},\"references\":[\"6e4b599c-1d09-46c1-83b7-d66df0337009X0\"],\"customLabel\":true},\"567b3cde-4bc5-4155-82bb-8c0f1097af27X0\":{\"label\":\"Part of Failure Count\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"result: FAILURE\",\"language\":\"kuery\"},\"customLabel\":true},\"567b3cde-4bc5-4155-82bb-8c0f1097af27\":{\"label\":\"Failure Count\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"count(kql='result: FAILURE')\",\"isFormulaBroken\":false},\"references\":[\"567b3cde-4bc5-4155-82bb-8c0f1097af27X0\"],\"customLabel\":true},\"8c65e9ca-7985-43ec-ae2e-7791dff2e7d3\":{\"label\":\"Timestamp\",\"dataType\":\"string\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ordinal\",\"sourceField\":\"@timestamp-original\",\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"76358e4d-51dc-4bb7-9763-45c422e51e5bX0\":{\"label\":\"Part of Started Tasks\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"result : STARTED\",\"language\":\"kuery\"},\"customLabel\":true},\"76358e4d-51dc-4bb7-9763-45c422e51e5bX1\":{\"label\":\"Part of Started Tasks\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"result : SUCCESS\",\"language\":\"kuery\"},\"customLabel\":true},\"76358e4d-51dc-4bb7-9763-45c422e51e5bX2\":{\"label\":\"Part of Started Tasks\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"result: FAILURE\",\"language\":\"kuery\"},\"customLabel\":true},\"76358e4d-51dc-4bb7-9763-45c422e51e5bX3\":{\"label\":\"Part of Started Tasks\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"Records\",\"filter\":{\"query\":\"result : PARTIAL_SUCCESS\",\"language\":\"kuery\"},\"customLabel\":true},\"76358e4d-51dc-4bb7-9763-45c422e51e5bX4\":{\"label\":\"Part of Started Tasks\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"subtract\",\"args\":[{\"type\":\"function\",\"name\":\"subtract\",\"args\":[{\"type\":\"function\",\"name\":\"subtract\",\"args\":[\"76358e4d-51dc-4bb7-9763-45c422e51e5bX0\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX1\"]},\"76358e4d-51dc-4bb7-9763-45c422e51e5bX2\"]},\"76358e4d-51dc-4bb7-9763-45c422e51e5bX3\"],\"location\":{\"min\":0,\"max\":132},\"text\":\"count(kql='result : STARTED') - count(kql='result : SUCCESS') - count(kql='result: FAILURE') - count(kql='result : PARTIAL_SUCCESS')\"}},\"references\":[\"76358e4d-51dc-4bb7-9763-45c422e51e5bX0\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX1\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX2\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX3\"],\"customLabel\":true},\"76358e4d-51dc-4bb7-9763-45c422e51e5b\":{\"label\":\"Ongoing Tasks\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"count(kql='result : STARTED') - count(kql='result : SUCCESS') - count(kql='result: FAILURE') - count(kql='result : PARTIAL_SUCCESS')\",\"isFormulaBroken\":false},\"references\":[\"76358e4d-51dc-4bb7-9763-45c422e51e5bX4\"],\"customLabel\":true}},\"columnOrder\":[\"73954412-33dd-4460-9ed0-25fb11272df8\",\"8c65e9ca-7985-43ec-ae2e-7791dff2e7d3\",\"76358e4d-51dc-4bb7-9763-45c422e51e5b\",\"6c1cf201-6770-49ee-9056-9bc1f009f5da\",\"6c1cf201-6770-49ee-9056-9bc1f009f5daX0\",\"567b3cde-4bc5-4155-82bb-8c0f1097af27\",\"567b3cde-4bc5-4155-82bb-8c0f1097af27X0\",\"6e4b599c-1d09-46c1-83b7-d66df0337009\",\"6e4b599c-1d09-46c1-83b7-d66df0337009X0\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX0\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX1\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX2\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX3\",\"76358e4d-51dc-4bb7-9763-45c422e51e5bX4\"],\"incompleteColumns\":{}}}}}}},\"enhancements\":{\"dynamicActions\":{\"events\":[{\"eventId\":\"78a728c0-3ead-4511-8742-5840b4e891b3\",\"triggers\":[\"FILTER_TRIGGER\"],\"action\":{\"factoryId\":\"DASHBOARD_TO_DASHBOARD_DRILLDOWN\",\"name\":\"Go to Dashboard\",\"config\":{\"useCurrentFilters\":true,\"useCurrentDateRange\":true}}}]}}}}]","timeRestore":false,"title":"Executions","version":1},"coreMigrationVersion":"7.15.2","id":"0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5","migrationVersion":{"dashboard":"7.15.0"},"references":[{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"d62f4d3d-027b-408d-b8ad-62404b90a057:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5","name":"d62f4d3d-027b-408d-b8ad-62404b90a057:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca","type":"index-pattern"},{"id":"994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5","name":"d62f4d3d-027b-408d-b8ad-62404b90a057:drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId","type":"dashboard"},{"id":"994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5","name":"d62f4d3d-027b-408d-b8ad-62404b90a057:drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId","type":"dashboard"}],"sort":[1714045118193,931],"type":"dashboard","updated_at":"2024-04-25T11:38:38.193Z","version":"WzIwNjUzLDEzXQ=="} +{"attributes":{"accessCount":1,"accessDate":1713774467694,"createDate":1713774391835,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:asc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data,sorting:(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,direction:desc))),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:edit)"},"coreMigrationVersion":"7.15.2","id":"10e336e54901315dfa88c7b31b0a46e0","references":[],"sort":[1713774467694,515],"type":"url","updated_at":"2024-04-22T08:27:47.694Z","version":"WzIwMTk4LDEzXQ=="} +{"attributes":{"accessCount":1,"accessDate":1714045528225,"createDate":1714045492705,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(%2773954412-33dd-4460-9ed0-25fb11272df8%27,%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27,%276c1cf201-6770-49ee-9056-9bc1f009f5da%27,%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27,%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27,%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27,%276e4b599c-1d09-46c1-83b7-d66df0337009%27,%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27),columns:(%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Failure%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result:%20FAILURE!%27)%27,isFormulaBroken:!f),references:!(%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27),scale:ratio),%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result:%20FAILURE%27),isBucketed:!f,label:%27Part%20of%20Failure%20Count%27,operationType:count,scale:ratio,sourceField:Records),%276c1cf201-6770-49ee-9056-9bc1f009f5da%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Success%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27),scale:ratio),%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20SUCCESS%27),isBucketed:!f,label:%27Part%20of%20count(kql%3D!%27result%20:%20SUCCESS!%27)%27,operationType:count,scale:ratio,sourceField:Records),%276e4b599c-1d09-46c1-83b7-d66df0337009%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Partial%20Success%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27),scale:ratio),%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20PARTIAL_SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Partial%20Success%20Count%27,operationType:count,scale:ratio,sourceField:Records),%2773954412-33dd-4460-9ed0-25fb11272df8%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(fallback:!t,type:alphabetical),orderDirection:asc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword),%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Ongoing%20Tasks%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20STARTED!%27)%20-%20count(kql%3D!%27result%20:%20SUCCESS!%27)%20-%20count(kql%3D!%27result:%20FAILURE!%27)%20-%20count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27),scale:ratio),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20STARTED%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result:%20FAILURE%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20PARTIAL_SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:math,params:(tinymathAst:(args:!((args:!((args:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27),name:subtract,type:function),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27),name:subtract,type:function),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27),location:(max:132,min:0),name:subtract,text:%27count(kql%3D!%27result%20:%20STARTED!%27)%20-%20count(kql%3D!%27result%20:%20SUCCESS!%27)%20-%20count(kql%3D!%27result:%20FAILURE!%27)%20-%20count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,type:function)),references:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27),scale:ratio),%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27:(customLabel:!t,dataType:string,isBucketed:!f,label:Timestamp,operationType:last_value,params:(sortField:%27@timestamp%27),scale:ordinal,sourceField:%27@timestamp-original%27)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:%2773954412-33dd-4460-9ed0-25fb11272df8%27,isTransposed:!f,width:297.66666666666663),(columnId:%276c1cf201-6770-49ee-9056-9bc1f009f5da%27,isTransposed:!f),(columnId:%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27,isTransposed:!f),(columnId:%276e4b599c-1d09-46c1-83b7-d66df0337009%27,isTransposed:!f),(columnId:%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,isTransposed:!f,width:213.66666666666666),(columnId:%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data,sorting:(columnId:%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,direction:desc))),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:16,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:48,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)&show-query-input=true&show-time-filter=true"},"coreMigrationVersion":"7.15.2","id":"2a517919d8081cb04ccd2c61a0e1c74a","references":[],"sort":[1714045528226,935],"type":"url","updated_at":"2024-04-25T11:45:28.226Z","version":"WzIwNjU4LDEzXQ=="} +{"attributes":{"accessCount":4,"accessDate":1713268236271,"createDate":1713268122764,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data)),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:edit)&show-time-filter=true"},"coreMigrationVersion":"7.15.2","id":"30f7ebeef8b48c64c79da05ff1588bba","references":[],"sort":[1713268236271,471],"type":"url","updated_at":"2024-04-16T11:50:36.271Z","version":"WzIwMTM5LDEzXQ=="} +{"attributes":{"accessCount":0,"accessDate":1713268354177,"createDate":1713268354177,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data)),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)"},"coreMigrationVersion":"7.15.2","id":"37a997d487481a422d52a3588547cb58","references":[],"sort":[1713268354177,473],"type":"url","updated_at":"2024-04-16T11:52:34.177Z","version":"WzIwMTQxLDEzXQ=="} +{"attributes":{"accessCount":1,"accessDate":1713271630902,"createDate":1713271613093,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:asc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data,sorting:(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,direction:desc))),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)&show-top-menu=true&show-query-input=true&show-time-filter=true"},"coreMigrationVersion":"7.15.2","id":"41b1fa963c2ae0c20701f43be4c3a7e6","references":[],"sort":[1713271630902,485],"type":"url","updated_at":"2024-04-16T12:47:10.902Z","version":"WzIwMTUzLDEzXQ=="} +{"attributes":{"buildNum":44266,"defaultIndex":"3f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5"},"coreMigrationVersion":"7.15.2","id":"7.15.2","migrationVersion":{"config":"7.13.0"},"references":[],"sort":[1713265882631,425],"type":"config","updated_at":"2024-04-16T11:11:22.631Z","version":"WzIwMDkzLDEzXQ=="} +{"attributes":{"accessCount":21,"accessDate":1714045364167,"createDate":1713271648005,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:asc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data,sorting:(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,direction:desc))),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)&show-query-input=true&show-time-filter=true"},"coreMigrationVersion":"7.15.2","id":"797a37b8b559f872a4933989160014cb","references":[],"sort":[1714045364167,932],"type":"url","updated_at":"2024-04-25T11:42:44.167Z","version":"WzIwNjU0LDEzXQ=="} +{"attributes":{"accessCount":0,"accessDate":1713268357275,"createDate":1713268357275,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(c7e3cb49-4024-4ae7-94ed-e4f26c211329,%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,%2747086a8d-d782-4a96-bf3b-74d085df7923%27,c65d2548-c412-43ab-a468-7f8872db19a2),columns:(%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Start%20Time%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:%27@timestamp-original%27),%2747086a8d-d782-4a96-bf3b-74d085df7923%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Error%20Status%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:3),scale:ordinal,sourceField:result.keyword),c65d2548-c412-43ab-a468-7f8872db19a2:(dataType:number,isBucketed:!f,label:%27Count%20of%20records%27,operationType:count,scale:ratio,sourceField:Records),c7e3cb49-4024-4ae7-94ed-e4f26c211329:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,type:column),orderDirection:desc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:c7e3cb49-4024-4ae7-94ed-e4f26c211329,isTransposed:!f),(columnId:%2727dda7b3-d9d1-4edb-86ae-fdde40d5f75f%27,isTransposed:!f),(columnId:%2747086a8d-d782-4a96-bf3b-74d085df7923%27,isTransposed:!f),(columnId:c65d2548-c412-43ab-a468-7f8872db19a2,hidden:!t,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data)),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:15,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:24,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)&show-time-filter=true"},"coreMigrationVersion":"7.15.2","id":"97e24daa9135b3f09857761fa490e663","references":[],"sort":[1713268357275,472],"type":"url","updated_at":"2024-04-16T11:52:37.275Z","version":"WzIwMTQwLDEzXQ=="} +{"attributes":{"accessCount":2,"accessDate":1714045457802,"createDate":1714045392621,"url":"/app/dashboards#/view/0ffb15e0-fbe5-11ee-b9d0-27e6ccfc8aa5?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now%2Fd,to:now%2Fd))&_a=(description:%27%27,filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,syncColors:!f,useMargins:!t),panels:!((embeddableConfig:(attributes:(references:!((id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-current-indexpattern,type:index-pattern),(id:%273f66f730-fbe1-11ee-b9d0-27e6ccfc8aa5%27,name:indexpattern-datasource-layer-a67e6635-b164-4c45-a563-bbe4ca458fca,type:index-pattern),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard),(id:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,name:%27drilldown:DASHBOARD_TO_DASHBOARD_DRILLDOWN:78a728c0-3ead-4511-8742-5840b4e891b3:dashboardId%27,type:dashboard)),state:(datasourceStates:(indexpattern:(layers:(a67e6635-b164-4c45-a563-bbe4ca458fca:(columnOrder:!(%2773954412-33dd-4460-9ed0-25fb11272df8%27,%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27,%276c1cf201-6770-49ee-9056-9bc1f009f5da%27,%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27,%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27,%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27,%276e4b599c-1d09-46c1-83b7-d66df0337009%27,%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27),columns:(%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Failure%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result:%20FAILURE!%27)%27,isFormulaBroken:!f),references:!(%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27),scale:ratio),%27567b3cde-4bc5-4155-82bb-8c0f1097af27X0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result:%20FAILURE%27),isBucketed:!f,label:%27Part%20of%20Failure%20Count%27,operationType:count,scale:ratio,sourceField:Records),%276c1cf201-6770-49ee-9056-9bc1f009f5da%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Success%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27),scale:ratio),%276c1cf201-6770-49ee-9056-9bc1f009f5daX0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20SUCCESS%27),isBucketed:!f,label:%27Part%20of%20count(kql%3D!%27result%20:%20SUCCESS!%27)%27,operationType:count,scale:ratio,sourceField:Records),%276e4b599c-1d09-46c1-83b7-d66df0337009%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Partial%20Success%20Count%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27),scale:ratio),%276e4b599c-1d09-46c1-83b7-d66df0337009X0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20PARTIAL_SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Partial%20Success%20Count%27,operationType:count,scale:ratio,sourceField:Records),%2773954412-33dd-4460-9ed0-25fb11272df8%27:(customLabel:!t,dataType:string,isBucketed:!t,label:%27Execution%20ID%27,operationType:terms,params:(missingBucket:!f,orderBy:(fallback:!t,type:alphabetical),orderDirection:asc,otherBucket:!t,size:5),scale:ordinal,sourceField:executionId.keyword),%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Ongoing%20Tasks%27,operationType:formula,params:(formula:%27count(kql%3D!%27result%20:%20STARTED!%27)%20-%20count(kql%3D!%27result%20:%20SUCCESS!%27)%20-%20count(kql%3D!%27result:%20FAILURE!%27)%20-%20count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,isFormulaBroken:!f),references:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27),scale:ratio),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20STARTED%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result:%20FAILURE%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27:(customLabel:!t,dataType:number,filter:(language:kuery,query:%27result%20:%20PARTIAL_SUCCESS%27),isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:count,scale:ratio,sourceField:Records),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX4%27:(customLabel:!t,dataType:number,isBucketed:!f,label:%27Part%20of%20Started%20Tasks%27,operationType:math,params:(tinymathAst:(args:!((args:!((args:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27),name:subtract,type:function),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27),name:subtract,type:function),%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27),location:(max:132,min:0),name:subtract,text:%27count(kql%3D!%27result%20:%20STARTED!%27)%20-%20count(kql%3D!%27result%20:%20SUCCESS!%27)%20-%20count(kql%3D!%27result:%20FAILURE!%27)%20-%20count(kql%3D!%27result%20:%20PARTIAL_SUCCESS!%27)%27,type:function)),references:!(%2776358e4d-51dc-4bb7-9763-45c422e51e5bX0%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX1%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX2%27,%2776358e4d-51dc-4bb7-9763-45c422e51e5bX3%27),scale:ratio),%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27:(customLabel:!t,dataType:string,isBucketed:!f,label:Timestamp,operationType:last_value,params:(sortField:%27@timestamp%27),scale:ordinal,sourceField:%27@timestamp-original%27)),incompleteColumns:())))),filters:!(),query:(language:kuery,query:%27%27),visualization:(columns:!((columnId:%2773954412-33dd-4460-9ed0-25fb11272df8%27,isTransposed:!f,width:297.66666666666663),(columnId:%276c1cf201-6770-49ee-9056-9bc1f009f5da%27,isTransposed:!f),(columnId:%27567b3cde-4bc5-4155-82bb-8c0f1097af27%27,isTransposed:!f),(columnId:%276e4b599c-1d09-46c1-83b7-d66df0337009%27,isTransposed:!f),(columnId:%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,isTransposed:!f,width:213.66666666666666),(columnId:%2776358e4d-51dc-4bb7-9763-45c422e51e5b%27,isTransposed:!f)),layerId:a67e6635-b164-4c45-a563-bbe4ca458fca,layerType:data,sorting:(columnId:%278c65e9ca-7985-43ec-ae2e-7791dff2e7d3%27,direction:desc))),title:%27%27,type:lens,visualizationType:lnsDatatable),enhancements:(dynamicActions:(events:!((action:(config:(dashboardId:%27994ee0b0-fbe5-11ee-b9d0-27e6ccfc8aa5%27,useCurrentDateRange:!t,useCurrentFilters:!t),factoryId:DASHBOARD_TO_DASHBOARD_DRILLDOWN,name:%27Go%20to%20Dashboard%27),eventId:%2778a728c0-3ead-4511-8742-5840b4e891b3%27,triggers:!(FILTER_TRIGGER)))))),gridData:(h:16,i:d62f4d3d-027b-408d-b8ad-62404b90a057,w:48,x:0,y:0),panelIndex:d62f4d3d-027b-408d-b8ad-62404b90a057,type:lens,version:%277.15.2%27)),query:(language:kuery,query:%27%27),tags:!(),timeRestore:!f,title:Executions,viewMode:view)"},"coreMigrationVersion":"7.15.2","id":"c8e40b39e912579ca0ccfd05c90d3302","references":[],"sort":[1714045457802,934],"type":"url","updated_at":"2024-04-25T11:44:17.802Z","version":"WzIwNjU3LDEzXQ=="} +{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":13,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/docker/kibana-data/index_template.txt b/docker/kibana-data/index_template.txt new file mode 100644 index 00000000..716f15bc --- /dev/null +++ b/docker/kibana-data/index_template.txt @@ -0,0 +1,24 @@ +PUT _template/ignore_above +{ + "index_patterns": [ + "fluentd*" + ], + "mappings": { + "dynamic_templates": [ + { + "strings_as_keyword": { + "match_mapping_type": "string", + "mapping": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 10000, + "type": "keyword" + } + } + } + } + } + ] + } +} \ No newline at end of file diff --git a/docker/log-server/Dockerfile b/docker/log-server/Dockerfile deleted file mode 100644 index 9095e30c..00000000 --- a/docker/log-server/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM openjdk:11-jre-slim - -ENV TOFHIR_HOME /usr/local/tofhir -RUN mkdir -p "$TOFHIR_HOME" -WORKDIR $TOFHIR_HOME - -COPY ./tofhir-log-server/target/tofhir-log-server-standalone.jar . -COPY ./docker/log-server/docker-entrypoint.sh . -RUN chmod +x docker-entrypoint.sh - -ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/docker/log-server/build.sh b/docker/log-server/build.sh deleted file mode 100644 index 196a84d0..00000000 --- a/docker/log-server/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -# Execute one of the following commands from the project.root.directory (../../) - -docker build -f docker/log-server/Dockerfile -t srdc/tofhir-log-server:latest . - diff --git a/docker/log-server/docker-entrypoint.sh b/docker/log-server/docker-entrypoint.sh deleted file mode 100644 index 1ddcebca..00000000 --- a/docker/log-server/docker-entrypoint.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -JAVA_CMD="java -Xms256m -Xmx3g -jar " - -# Configure application.conf path -if [ ! -z "$APP_CONF_FILE" ]; then - JAVA_CMD+="-Dconfig.file=$APP_CONF_FILE " -fi - -# Configure logback configuration file -if [ ! -z "$LOGBACK_CONF_FILE" ]; then - JAVA_CMD+="-Dlogback.configurationFile=$LOGBACK_CONF_FILE " -fi - -# Configure Spark -if [ ! -z "$SPARK_APPNAME" ]; then - JAVA_CMD+="-Dspark.app-name=$SPARK_APPNAME " -fi -if [ ! -z "$SPARK_MASTER" ]; then - JAVA_CMD+="-Dspark.master=$SPARK_MASTER " -fi - -# Configure tofhir-server web server -if [ ! -z "$WEBSERVER_HOST" ]; then - JAVA_CMD+="-Dwebserver.host=$WEBSERVER_HOST " -fi -if [ ! -z "$WEBSERVER_PORT" ]; then - JAVA_CMD+="-Dwebserver.port=$WEBSERVER_PORT " -fi -if [ ! -z "$WEBSERVER_BASEURI" ]; then - JAVA_CMD+="-Dwebserver.base-uri=$WEBSERVER_BASEURI " -fi - -# Delay the execution for this amount of seconds -if [ ! -z "$DELAY_EXECUTION" ]; then - sleep $DELAY_EXECUTION -fi - -# Finally, tell which jar to run -JAVA_CMD+="tofhir-log-server-standalone.jar" - -eval $JAVA_CMD "$@" diff --git a/docker/server/docker-entrypoint.sh b/docker/server/docker-entrypoint.sh index d8456c61..858cd04d 100644 --- a/docker/server/docker-entrypoint.sh +++ b/docker/server/docker-entrypoint.sh @@ -74,11 +74,6 @@ if [ ! -z "$WEBSERVER_BASEURI" ]; then JAVA_CMD+="-Dwebserver.base-uri=$WEBSERVER_BASEURI " fi -# Configure log service -if [ ! -z "$LOG_SERVICE_ENDPOINT" ]; then - JAVA_CMD+="-Dlog-service.endpoint=$LOG_SERVICE_ENDPOINT " -fi - # Delay the execution for this amount of seconds if [ ! -z "$DELAY_EXECUTION" ]; then sleep $DELAY_EXECUTION diff --git a/pom.xml b/pom.xml index 68aa6371..bac1d126 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,6 @@ tofhir-engine tofhir-server - tofhir-log-server tofhir-common tofhir-server-common tofhir-rxnorm @@ -101,6 +100,8 @@ 1.1-SNAPSHOT 1.0-SNAPSHOT 3.7.0-M11 + 1.8.8 + 0.3.4 2.15.1 3.2.17 3.5.1 @@ -280,6 +281,22 @@ + + + + com.sndyuk + logback-more-appenders + ${logback-more-appenders.version} + + + + org.fluentd + fluent-logger + ${fluent-logger.version} + + @@ -418,11 +435,6 @@ tofhir-server-common_2.13 ${project.version} - - io.onfhir - tofhir-log-server-common_2.13 - ${project.version} - io.onfhir tofhir-engine_2.13 diff --git a/readme-assets/kibana-execution-details.png b/readme-assets/kibana-execution-details.png new file mode 100644 index 00000000..2e2e6984 Binary files /dev/null and b/readme-assets/kibana-execution-details.png differ diff --git a/readme-assets/kibana-executions-dashboard.png b/readme-assets/kibana-executions-dashboard.png new file mode 100644 index 00000000..0cd2b998 Binary files /dev/null and b/readme-assets/kibana-executions-dashboard.png differ diff --git a/readme-assets/kibana-go-to-dashboard.png b/readme-assets/kibana-go-to-dashboard.png new file mode 100644 index 00000000..54511e7a Binary files /dev/null and b/readme-assets/kibana-go-to-dashboard.png differ diff --git a/readme-assets/kibana-index-template.png b/readme-assets/kibana-index-template.png new file mode 100644 index 00000000..f4be21e5 Binary files /dev/null and b/readme-assets/kibana-index-template.png differ diff --git a/readme-assets/kibana-saved-objects.png b/readme-assets/kibana-saved-objects.png new file mode 100644 index 00000000..dde21417 Binary files /dev/null and b/readme-assets/kibana-saved-objects.png differ diff --git a/readme-assets/module-component-diagram.png b/readme-assets/module-component-diagram.png index 5a3d3c04..951976cc 100644 Binary files a/readme-assets/module-component-diagram.png and b/readme-assets/module-component-diagram.png differ diff --git a/tofhir-engine/pom.xml b/tofhir-engine/pom.xml index b139f6e1..4d29132d 100644 --- a/tofhir-engine/pom.xml +++ b/tofhir-engine/pom.xml @@ -200,6 +200,15 @@ org.apache.spark spark-sql_${scala.binary.version} + + + com.sndyuk + logback-more-appenders + + + org.fluentd + fluent-logger + diff --git a/tofhir-engine/src/main/resources/logback.xml b/tofhir-engine/src/main/resources/logback.xml index 0a6d1fc6..e2817f79 100644 --- a/tofhir-engine/src/main/resources/logback.xml +++ b/tofhir-engine/src/main/resources/logback.xml @@ -45,6 +45,9 @@ [ignore] + + logs/tofhir-mappings.%i.log.zip @@ -57,6 +60,19 @@ + + + + localhost + 24224 + + + INFO + + + true + + @@ -69,6 +85,7 @@ + diff --git a/tofhir-engine/src/main/scala/io/tofhir/engine/data/write/SinkHandler.scala b/tofhir-engine/src/main/scala/io/tofhir/engine/data/write/SinkHandler.scala index 8fdb7371..f58a6109 100644 --- a/tofhir-engine/src/main/scala/io/tofhir/engine/data/write/SinkHandler.scala +++ b/tofhir-engine/src/main/scala/io/tofhir/engine/data/write/SinkHandler.scala @@ -90,18 +90,18 @@ object SinkHandler { //Log the job result val jobResult = FhirMappingJobResult(mappingJobExecution, mappingUrl, numOfInvalids, numOfNotMapped, numOfWritten, numOfNotWritten) - logger.info(jobResult.toLogstashMarker, jobResult.toString) + logger.info(jobResult.toMapMarker, jobResult.toString) // Log the mapping and invalid input errors if (numOfNotMapped > 0 || numOfInvalids > 0) { mappingErrors.union(invalidInputs).foreach(r => - logger.warn(r.copy(executionId = Some(mappingJobExecution.id)).toLogstashMarker, + logger.warn(r.copy(executionId = Some(mappingJobExecution.id)).toMapMarker, r.copy(executionId = Some(mappingJobExecution.id)).toString) ) } if (numOfNotWritten > 0) notWrittenResources.forEach(r => - logger.warn(r.copy(executionId = Some(mappingJobExecution.id)).toLogstashMarker, + logger.warn(r.copy(executionId = Some(mappingJobExecution.id)).toMapMarker, r.copy(executionId = Some(mappingJobExecution.id)).toString) ) } diff --git a/tofhir-engine/src/main/scala/io/tofhir/engine/execution/RunningJobRegistry.scala b/tofhir-engine/src/main/scala/io/tofhir/engine/execution/RunningJobRegistry.scala index 0b3302ce..0a7dd62d 100644 --- a/tofhir-engine/src/main/scala/io/tofhir/engine/execution/RunningJobRegistry.scala +++ b/tofhir-engine/src/main/scala/io/tofhir/engine/execution/RunningJobRegistry.scala @@ -344,6 +344,16 @@ class RunningJobRegistry(spark: SparkSession) { runningTasks.get(jobId).map(_.keySet).getOrElse(Set.empty).toSet } + /** + * Gets scheduled executions for the given job + * + * @param jobId Identifier of the job + * @return A set of execution ids + */ + def getScheduledExecutions(jobId: String): Set[String] = { + scheduledTasks.get(jobId).map(_.keySet).getOrElse(Set.empty).toSet + } + /** * Checks if a job with the given execution ID is scheduled. * diff --git a/tofhir-engine/src/main/scala/io/tofhir/engine/logback/MapMarkerToLogstashMarkerEncoder.scala b/tofhir-engine/src/main/scala/io/tofhir/engine/logback/MapMarkerToLogstashMarkerEncoder.scala new file mode 100644 index 00000000..716d5bf9 --- /dev/null +++ b/tofhir-engine/src/main/scala/io/tofhir/engine/logback/MapMarkerToLogstashMarkerEncoder.scala @@ -0,0 +1,101 @@ +package io.tofhir.engine.logback + +import ch.qos.logback.classic.spi.{ILoggingEvent, IThrowableProxy, LoggerContextVO} +import ch.qos.logback.more.appenders.marker.MapMarker +import net.logstash.logback.encoder.LogstashEncoder +import net.logstash.logback.marker.LogstashMarker +import net.logstash.logback.marker.Markers.appendEntries + +import java.util + +/** + * An encoder for converting MapMarker objects to LogstashMarker objects. + */ +class MapMarkerToLogstashMarkerEncoder extends LogstashEncoder { + + /** + * Encodes the logging event by converting MapMarker to LogstashMarker and then calling super.encode(). + * + * @param event The logging event to encode. + * @return The encoded logging event as a byte array. + */ + override def encode(event: ILoggingEvent): Array[Byte] = { + val modifiedEvent = modifyEventWithLogstashMarker(event) + super.encode(modifiedEvent) + } + + /** + * Modifies the logging event with a LogstashMarker if it contains a MapMarker. + * + * @param event The logging event to modify. + * @return The modified logging event. + */ + private def modifyEventWithLogstashMarker(event: ILoggingEvent): ILoggingEvent = { + val map = getMapMarkerData(event.getMarker) + if (map.isDefined) { + val logstashMarker = toLogstashMarker(map.get) + new ModifiedLoggingEvent(event, logstashMarker) + } else { + event + } + } + + /** + * Extracts the data from a MapMarker. + * + * @param marker The marker to extract data from. + * @return An Optional containing the marker data if the marker is a MapMarker, otherwise empty. + */ + private def getMapMarkerData(marker: org.slf4j.Marker): Option[util.Map[String, _]] = { + marker match { + case m: MapMarker => Some(m.getMap) + case _ => None + } + } + + /** + * Converts a Map of marker data to a LogstashMarker. + * + * @param map The map of marker data. + * @return The corresponding LogstashMarker. + */ + private def toLogstashMarker(map: util.Map[String, _]): LogstashMarker = { + appendEntries(map) + } + + /** + * A wrapper class for the logging event that delegates to the original event but with a different marker. + */ + private class ModifiedLoggingEvent(originalEvent: ILoggingEvent, marker: LogstashMarker) extends ILoggingEvent { + override def getMarker: org.slf4j.Marker = marker + + // Implement other methods by delegating to the original event + override def getTimeStamp: Long = originalEvent.getTimeStamp + + override def getLevel: ch.qos.logback.classic.Level = originalEvent.getLevel + + override def getThreadName: String = originalEvent.getThreadName + + override def getMessage: String = originalEvent.getMessage + + override def getArgumentArray: Array[AnyRef] = originalEvent.getArgumentArray + + override def getFormattedMessage: String = originalEvent.getFormattedMessage + + override def getLoggerName: String = originalEvent.getLoggerName + + override def getLoggerContextVO: LoggerContextVO = originalEvent.getLoggerContextVO + + override def getThrowableProxy: IThrowableProxy = originalEvent.getThrowableProxy + + override def getCallerData: Array[StackTraceElement] = originalEvent.getCallerData + + override def hasCallerData: Boolean = originalEvent.hasCallerData + + override def getMDCPropertyMap: util.Map[String, String] = originalEvent.getMDCPropertyMap + + override def getMdc: util.Map[String, String] = originalEvent.getMdc + + override def prepareForDeferredProcessing(): Unit = originalEvent.prepareForDeferredProcessing() + } +} diff --git a/tofhir-engine/src/main/scala/io/tofhir/engine/mapping/FhirMappingJobManager.scala b/tofhir-engine/src/main/scala/io/tofhir/engine/mapping/FhirMappingJobManager.scala index 03bfe318..bab52142 100644 --- a/tofhir-engine/src/main/scala/io/tofhir/engine/mapping/FhirMappingJobManager.scala +++ b/tofhir-engine/src/main/scala/io/tofhir/engine/mapping/FhirMappingJobManager.scala @@ -62,7 +62,7 @@ class FhirMappingJobManager( mappingJobExecution.mappingTasks.foldLeft(Future((): Unit)) { (f, task) => // Initial empty Future f.flatMap { _ => // Execute the Futures in the Sequence consecutively (not in parallel) val jobResult = FhirMappingJobResult(mappingJobExecution, Some(task.mappingRef)) - logger.info(jobResult.toLogstashMarker, jobResult.toString) + logger.info(jobResult.toMapMarker, jobResult.toString) readSourceExecuteAndWriteInBatches(mappingJobExecution.copy(mappingTasks = Seq(task)), sourceSettings, fhirWriter, terminologyServiceSettings, identityServiceSettings, timeRange) @@ -70,7 +70,7 @@ class FhirMappingJobManager( // Check whether the job is stopped case se: SparkThrowable if se.getMessage.contains("cancelled part of cancelled job group") => val jobResult = FhirMappingJobResult(mappingJobExecution, Some(task.mappingRef), status = Some(FhirMappingJobResult.STOPPED)) - logger.info(jobResult.toLogstashMarker, jobResult.toString) + logger.info(jobResult.toMapMarker, jobResult.toString) throw FhirMappingJobStoppedException(s"Execution '${mappingJobExecution.id}' of job '${mappingJobExecution.job.id}' in project ${mappingJobExecution.projectId}' terminated manually!") // Exceptions from Spark executors are wrapped inside a SparkException, which are caught below case se: SparkThrowable => @@ -78,17 +78,17 @@ class FhirMappingJobManager( // log the mapping job result and exception for the errors encountered while reading the schema or writing the FHIR Resources case _ => val jobResult = FhirMappingJobResult(mappingJobExecution, Some(task.mappingRef), status = Some(FhirMappingJobResult.FAILURE)) - logger.error(jobResult.toLogstashMarker, jobResult.toString, se) + logger.error(jobResult.toMapMarker, jobResult.toString, se) } // Pass the stop exception to the upstream Futures in the chain laid out by foldLeft above case t: FhirMappingJobStoppedException => val jobResult = FhirMappingJobResult(mappingJobExecution, Some(task.mappingRef), status = Some(FhirMappingJobResult.SKIPPED)) - logger.info(jobResult.toLogstashMarker, jobResult.toString) + logger.info(jobResult.toMapMarker, jobResult.toString) throw t case e: Throwable => // log the mapping job result and exception val jobResult = FhirMappingJobResult(mappingJobExecution, Some(task.mappingRef), status = Some(FhirMappingJobResult.FAILURE)) - logger.error(jobResult.toLogstashMarker, jobResult.toString, e) + logger.error(jobResult.toMapMarker, jobResult.toString, e) } } map { _ => logger.debug(s"MappingJob execution finished for MappingJob: ${mappingJobExecution.job.id}.") } } @@ -115,7 +115,7 @@ class FhirMappingJobManager( .map(t => { logger.debug(s"Streaming mapping job ${mappingJobExecution.job.id}, mapping url ${t.mappingRef} is started and waiting for the data...") val jobResult = FhirMappingJobResult(mappingJobExecution, Some(t.mappingRef)) - logger.info(jobResult.toLogstashMarker, jobResult.toString) + logger.info(jobResult.toMapMarker, jobResult.toString) // Construct a tuple of (mapping url, Future[StreamingQuery]) t.mappingRef -> @@ -126,7 +126,7 @@ class FhirMappingJobManager( .recover { case e: Throwable => val jobResult = FhirMappingJobResult(mappingJobExecution, Some(t.mappingRef), status = Some(FhirMappingJobResult.FAILURE)) - logger.error(jobResult.toLogstashMarker, jobResult.toString,e) + logger.error(jobResult.toMapMarker, jobResult.toString,e) throw e } }) diff --git a/tofhir-engine/src/main/scala/io/tofhir/engine/model/FhirMappingJobResult.scala b/tofhir-engine/src/main/scala/io/tofhir/engine/model/FhirMappingJobResult.scala index f9be2d41..2dd1202d 100644 --- a/tofhir-engine/src/main/scala/io/tofhir/engine/model/FhirMappingJobResult.scala +++ b/tofhir-engine/src/main/scala/io/tofhir/engine/model/FhirMappingJobResult.scala @@ -1,8 +1,6 @@ package io.tofhir.engine.model -import net.logstash.logback.marker.LogstashMarker -import net.logstash.logback.marker.Markers.{append} - +import ch.qos.logback.more.appenders.marker.MapMarker /** * Result of a batch mapping job execution * @@ -52,20 +50,26 @@ case class FhirMappingJobResult(mappingJobExecution: FhirMappingJobExecution, } /** + * Converts the mapping job execution to a MapMarker object. * - * @return + * @return The MapMarker representing the mapping job execution. */ - def toLogstashMarker: LogstashMarker = { - append("jobId", mappingJobExecution.job.id) - .and(append("projectId", mappingJobExecution.projectId) - .and(append("executionId", mappingJobExecution.id) - .and(append("mappingUrl", mappingUrl.orElse(null)) - .and(append("result", result) - .and(append("numOfInvalids", numOfInvalids) - .and(append("numOfNotMapped", numOfNotMapped) - .and(append("numOfFhirResources", numOfFhirResources) - .and(append("numOfFailedWrites", numOfFailedWrites) - .and(append("eventId", eventId)))))))))) + def toMapMarker: MapMarker = { + // create a new HashMap to store the marker attributes + val markerMap: java.util.Map[String, Any] = new java.util.HashMap[String, Any]() + // add attributes to the marker map + markerMap.put("jobId", mappingJobExecution.job.id) + markerMap.put("projectId", mappingJobExecution.projectId) + markerMap.put("executionId", mappingJobExecution.id) + markerMap.put("mappingUrl", mappingUrl.orNull) + markerMap.put("result", result) + markerMap.put("numOfInvalids", numOfInvalids) + markerMap.put("numOfNotMapped", numOfNotMapped) + markerMap.put("numOfFhirResources", numOfFhirResources) + markerMap.put("numOfFailedWrites", numOfFailedWrites) + markerMap.put("eventId", eventId) + // create a new MapMarker using the marker map + new MapMarker("marker", markerMap) } } diff --git a/tofhir-engine/src/main/scala/io/tofhir/engine/model/FhirMappingResult.scala b/tofhir-engine/src/main/scala/io/tofhir/engine/model/FhirMappingResult.scala index e6aae117..49314a14 100644 --- a/tofhir-engine/src/main/scala/io/tofhir/engine/model/FhirMappingResult.scala +++ b/tofhir-engine/src/main/scala/io/tofhir/engine/model/FhirMappingResult.scala @@ -1,9 +1,8 @@ package io.tofhir.engine.model -import net.logstash.logback.marker.LogstashMarker +import ch.qos.logback.more.appenders.marker.MapMarker import java.sql.Timestamp -import net.logstash.logback.marker.Markers._ /** @@ -38,25 +37,29 @@ case class FhirMappingResult( } /** + * Converts the FhirMappingResult to a MapMarker. * - * @return + * @return The MapMarker object representing the FhirMappingResult. */ - def toLogstashMarker:LogstashMarker = { - val marker:LogstashMarker = - append("jobId", jobId) - .and(append("executionId", executionId.getOrElse("")) - .and(append("mappingUrl", mappingUrl) - .and(append("mappingExpr", mappingExpr.orElse(null)) - .and(appendRaw("source", source.get) - .and(append("errorCode", error.get.code) - .and(append("errorDesc", error.get.description) - .and(append("errorExpr", error.get.expression.orElse(null)) - .and(append("eventId", eventId))))))))) - - if(mappedResource.isDefined && error.get.code == FhirMappingErrorCodes.INVALID_RESOURCE) - marker.and(appendRaw("mappedResource", mappedResource.get)) - else - marker + def toMapMarker: MapMarker = { + // create a new HashMap to store the marker attributes + val markerMap: java.util.Map[String, Any] = new java.util.HashMap[String, Any]() + // add attributes to the marker map + markerMap.put("jobId", jobId) + markerMap.put("executionId", executionId.getOrElse("")) + markerMap.put("mappingUrl", mappingUrl) + markerMap.put("mappingExpr", mappingExpr.orElse(null)) + markerMap.put("source", source.get) + markerMap.put("errorCode", error.get.code) + markerMap.put("errorDesc", error.get.description) + markerMap.put("errorExpr", error.get.expression.orElse(null)) + markerMap.put("eventId", eventId) + // create a new MapMarker using the marker map + val marker: MapMarker = new MapMarker("marker", markerMap) + // add mappedResource to the marker map if error code is INVALID_RESOURCE + if (mappedResource.isDefined && error.get.code == FhirMappingErrorCodes.INVALID_RESOURCE) + markerMap.put("mappedResource", mappedResource.get) + marker } } diff --git a/tofhir-log-server/api.yaml b/tofhir-log-server/api.yaml deleted file mode 100644 index 8224158b..00000000 --- a/tofhir-log-server/api.yaml +++ /dev/null @@ -1,283 +0,0 @@ -openapi: 3.0.2 -info: - description: "Lists the REST APIs provided by a toFHIR log server" - version: "1.1" - title: "toFHIR-log-server REST API" - -servers: - - url: "http://localhost:8086/tofhir-logs" - description: Local deployment of the toFHIR log server - - -tags: - - name: "Execution" - description: "Represents a logical grouping for project entities" - - -paths: - # Beginning of the Execution tag - /projects/{projectId}/jobs/{jobId}/executions: - get: - tags: - - Execution - summary: Gets executions of the mapping job - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - # Filter parameters for lazy load - - name: page - in: query - required: true - description: requested page of the executions - schema: - type: integer - example: 1 - - name: dateBefore - in: query - required: false - description: Executions started before this date - schema: - type: string - - name: dateAfter - in: query - required: false - description: Executions started after this date - schema: - type: string - - name: errorStatuses - in: query - required: false - description: Request type of executions written as comma seperated - schema: - type: string - - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/Execution" - - /projects/{projectId}/jobs/{jobId}/executions/{executionId}: - get: - tags: - - Execution - summary: Gets the details of the specified execution - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - - $ref: "#/components/parameters/executionId" - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/Execution" - '404': - $ref: "#/components/responses/404NotFound" - - /projects/{projectId}/jobs/{jobId}/executions/{executionId}/logs: - get: - tags: - - Execution - summary: Gets execution logs - parameters: - - $ref: "#/components/parameters/projectId" - - $ref: "#/components/parameters/jobId" - - $ref: "#/components/parameters/executionId" - responses: - '200': - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/Log" - '404': - $ref: "#/components/responses/404NotFound" - - -components: - schemas: - Execution: - type: object - description: Executions of a mapping job - properties: - id: - type: string - description: "Identifier of the execution" - example: "c531d5b0-40fc-42a9-8f2d-c6105e3a5d39" - mappingUrls: - type: array - items: - type: string - description: "Url of the mapping runned in the execution" - example: "https://aiccelerate.eu/fhir/mappings/pilot1/anesthesia-observations-mapping" - startTime: - type: string - description: "Start time of the execution" - example: "2024-01-26T10:19:39.538+03:00" - errorStatus: - type: string - description: "Result of the the mapping job" - enum: - - "SUCCESS" - - "FAILURE" - - "PARTIAL_SUCCESS" - - "STARTED" - Log: - type: object - description: "Logs of the individual executions" - properties: - "@timestamp": - type: string - description: "Timestamp of the event" - example: "2024-01-26T10:19:39.538+03:00" - errorCode: - type: string - description: "Error code, if any" - errorDesc: - type: string - description: "Error description, if any" - errorExpr: - type: string - description: "Expression that causes error, if any" - eventId: - type: string - description: "Identifier of the event" - example: "MAPPING_JOB_RESULT" - executionId: - type: string - description: "Identifier of the execution" - example: "c531d5b0-40fc-42a9-8f2d-c6105e3a5d39" - jobId: - type: string - description: "Identifier of the job" - example: "pilot1" - level: - type: string - description: "Level of the log" - example: "INFO" - logger_name: - type: string - description: "Name of the logger" - example: "io.tofhir.engine.mapping.FhirMappingJobManager" - mappedResource: - type: string - description: "Mapped resource, if any" - mappingExpr: - type: string - description: "Mapping expression, if any" - mappingUrl: - type: string - description: "URL of the mapping" - example: "https://aiccelerate.eu/fhir/mappings/pilot1/anesthesia-observations-mapping" - message: - type: string - description: "Log message" - example: "toFHIR batch mapping result (STARTED) for execution 'c531d5b0-40fc-42a9-8f2d-c6105e3a5d39' of job 'pilot1' in project 'pilot1'" - numOfFailedWrites: - type: integer - description: "Number of failed writes, -1 if there is an error" - example: -1 - numOfFhirResources: - type: integer - description: "Number of written FHIR resources, -1 if there is an error" - example: -1 - numOfInvalids: - type: integer - description: "Number of invalid rows, -1 if there is an error" - example: -1 - numOfNotMapped: - type: integer - description: "Number of not mapped rows, -1 if there is an error" - example: -1 - projectId: - type: string - description: "Identifier of the project" - example: "pilot1" - result: - type: string - description: "Result of the mapping job" - example: "STARTED" - source: - type: string - description: "Source of the error, if any" - stack_trace: - type: string - description: "Stack trace of the error, if any" - error_logs: - type: array - description: "Logs of the errors, if any" - items: - $ref: "#/components/schemas/ErrorLog" - ErrorLog: - type: object - description: "Logs of the errors" - properties: - errorCode: - type: string - description: "Error code" - example: "mapping_error" - errorDesc: - type: string - description: "Error description" - example: "FHIR path expression returns empty although value is not marked as optional! Please use '?' mark in placeholder e.g. {{? }} or correct your expression" - message: - type: string - description: "Detailed error message" - example: "Mapping failure (mapping_error) for job 'pilot1' and mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/condition-mapping' within expression 'Expression: result. Error: FHIR path expression returns empty although value is not marked as optional! Please use '?' mark in placeholder e.g. {{? }} or correct your expression' execution 'c531d5b0-40fc-42a9-8f2d-c6105e3a5d39'!\n\tSource: {\"pid\":\"p1\",\"code\":\"J13\",\"codeDisplay\":\"Pneumonia due to Streptococcus pneumoniae\",\"onsetDateTime\":\"2012-10-15\",\"abatementDateTime\":null,\"encounterId\":null,\"diagnosisType\":\"main\",\"isProlonged\":true,\"certainity\":\"confirmed\",\"asserter\":null}\n\tError: FHIR path expression returns empty although value is not marked as optional! Please use '?' mark in placeholder e.g. {{? }} or correct your expression\n\tExpression: Hello" - mappingUrl: - type: string - description: "URL of the mapping" - example: "https://aiccelerate.eu/fhir/mappings/pilot1/condition-mapping" - - # HTTP responses for requests - responses: - 400BadRequest: - description: "Bad or Invalid request" - 409AlreadyExists: - description: "Given object already exists" - 404NotFound: - description: "Given resource does not exist" - - # Path and query parameters for requests - parameters: - projectId: - in: path - name: "projectId" - required: true - schema: - type: string - format: uuid - pattern: "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" - description: "Identifier of the project" - - jobId: - in: path - name: "jobId" - schema: - type: string - required: true - description: "Identifier of the job" - - executionId: - name: executionId - in: path - required: true - schema: - type: string - - mappingUrl: - in: path - name: "mappingUrl" - schema: - type: string - format: url - required: true - description: "Url of a mapping" \ No newline at end of file diff --git a/tofhir-log-server/pom.xml b/tofhir-log-server/pom.xml deleted file mode 100644 index 1137487e..00000000 --- a/tofhir-log-server/pom.xml +++ /dev/null @@ -1,162 +0,0 @@ - - - 4.0.0 - - - io.onfhir - tofhir_2.13 - ${revision} - - - tofhir-log-server_2.13 - jar - - - src/main/scala - src/test/scala - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-surefire-plugin - - - org.scalatest - scalatest-maven-plugin - - - - src/test/resources/log-sample.log - - - - - org.apache.maven.plugins - maven-shade-plugin - - - make-assembly - package - - shade - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - false - tofhir-log-server-standalone - - - - io.tofhir.log.server.Boot - - - - - reference.conf - - - - - - - - - - - - - - - - - org.scala-lang - scala-library - - - - - com.typesafe.scala-logging - scala-logging_${scala.binary.version} - - - - - ch.qos.logback - logback-classic - - - ch.qos.logback - logback-core - - - - - com.typesafe - config - - - - - org.json4s - json4s-jackson_${scala.binary.version} - - - - - com.typesafe.akka - akka-http_${scala.binary.version} - - - com.typesafe.akka - akka-http-core_${scala.binary.version} - - - com.typesafe.akka - akka-http-caching_${scala.binary.version} - - - com.typesafe.akka - akka-actor-typed_${scala.binary.version} - - - com.typesafe.akka - akka-stream-typed_${scala.binary.version} - - - - - io.onfhir - tofhir-engine_2.13 - - - io.onfhir - tofhir-server-common - - - - - org.scalatest - scalatest_${scala.binary.version} - test - - - - - diff --git a/tofhir-log-server/src/main/resources/application.conf b/tofhir-log-server/src/main/resources/application.conf deleted file mode 100644 index 3f11057a..00000000 --- a/tofhir-log-server/src/main/resources/application.conf +++ /dev/null @@ -1,54 +0,0 @@ -webserver = { - # Hostname that toFHIR log server will work. Using 0.0.0.0 will bind the server to both localhost and the IP of the server that you deploy it. - host = 0.0.0.0 - - # Port to listen - port = 8086 - - # Base Uri for server e.g. With this default configuration, the root path of toFHIR log server will be http://localhost:8085/tofhir-logs - base-uri = tofhir-logs - - ssl { - # Path to the java keystore for enabling ssl for toFHIR server, use null to disable ssl - keystore = null - # Password of the keystore for enabling ssl for toFHIR server - password = null - } -} - -# Spark configurations -spark = { - app.name = "AICCELERATE Data Integration Suite" - master = "local[1]" -} - -akka = { - daemonic = "on" - - # Configurations for Akka HTTP - http = { - parsing = { - max-header-value-length = 82k - } - server = { - # Header for server - server-header = toFHIR Log Server - parsing{ - uri-parsing-mode = relaxed - } - # Request timeout for all REST services - request-timeout = 60 s - # Maximum inactivity time of a given HTTP connection - idle-timeout = 60 s - # Should be on in order to get IP address of the clients for audits - remote-address-header = on - } - } -} - -tofhir = { - log-server { - # The file that contains results of mapping executions - filepath = "logs/tofhir-mappings.log" - } -} diff --git a/tofhir-log-server/src/main/resources/logback.xml b/tofhir-log-server/src/main/resources/logback.xml deleted file mode 100644 index 79844906..00000000 --- a/tofhir-log-server/src/main/resources/logback.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - %d{yyyy-MM-dd_HH:mm:ss.SSS} [%thread] %-5level %logger{5} - %msg%n - - - - - - logs/tofhir-log-server.log - - %d{yyyy-MM-dd_HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - logs/tofhir-log-server.%i.log.zip - 1 - 10 - - - - 10MB - - - - - - - - - - - - - - - - - diff --git a/tofhir-log-server/src/main/scala/io/tofhir/log/server/Boot.scala b/tofhir-log-server/src/main/scala/io/tofhir/log/server/Boot.scala deleted file mode 100644 index bc9bbb07..00000000 --- a/tofhir-log-server/src/main/scala/io/tofhir/log/server/Boot.scala +++ /dev/null @@ -1,5 +0,0 @@ -package io.tofhir.log.server - -object Boot extends App { - ToFhirLogServer.start() -} diff --git a/tofhir-log-server/src/main/scala/io/tofhir/log/server/ToFhirLogHttpServer.scala b/tofhir-log-server/src/main/scala/io/tofhir/log/server/ToFhirLogHttpServer.scala deleted file mode 100644 index 0fc71d4d..00000000 --- a/tofhir-log-server/src/main/scala/io/tofhir/log/server/ToFhirLogHttpServer.scala +++ /dev/null @@ -1,67 +0,0 @@ -package io.tofhir.log.server - -import akka.Done -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import akka.http.scaladsl.server.Route -import com.typesafe.scalalogging.LazyLogging -import io.tofhir.server.common.config.WebServerConfig - -import java.util.concurrent.TimeUnit -import scala.concurrent.duration.{Duration, FiniteDuration} -import scala.concurrent._ -import scala.io.StdIn -import scala.util.{Failure, Success} - -object ToFhirLogHttpServer extends LazyLogging { - - def start(route: Route, webServerConfig: WebServerConfig)(implicit actorSystem: ActorSystem): Unit = { - implicit val executionContext: ExecutionContext = actorSystem.dispatcher - - val serverBindingFuture = Http().newServerAt(webServerConfig.serverHost, webServerConfig.serverPort).bind(route) - .map(serverBinding => { - serverBinding.addToCoordinatedShutdown(hardTerminationDeadline = FiniteDuration(10, TimeUnit.SECONDS)) - serverBinding.whenTerminated onComplete { - case Success(_) => - logger.info("Closing toFHIR HTTP server...") - actorSystem.terminate() - logger.info("toFHIR HTTP server gracefully terminated.") - case Failure(exception) => - logger.error("Problem while gracefully terminating toFHIR HTTP server!", exception) - } - serverBinding - }) - - var serverBinding: Option[Http.ServerBinding] = None - try { - serverBinding = Some(Await.result(serverBindingFuture, FiniteDuration(10L, TimeUnit.SECONDS))) - logger.info(s"tofHIR server ready at ${webServerConfig.serverHost}:${webServerConfig.serverPort}") - } catch { - case e: Exception => - logger.error("Problem while binding to the given HTTP address and port!", e) - actorSystem.terminate() - } - - //Wait for a shutdown signal - Await.ready(waitForShutdownSignal(), Duration.Inf) - serverBinding.get.terminate(FiniteDuration.apply(10L, TimeUnit.SECONDS)) - } - - protected def waitForShutdownSignal()(implicit executionContext: ExecutionContext): Future[Done] = { - val promise = Promise[Done]() - sys.addShutdownHook { - promise.trySuccess(Done) - } - Future { - blocking { - do { - val line = StdIn.readLine("Write 'q' or 'quit' to stop the server...\n") - if (line.equalsIgnoreCase("quit")) - promise.trySuccess(Done) - } while (!promise.isCompleted) - } - } - promise.future - } - -} diff --git a/tofhir-log-server/src/main/scala/io/tofhir/log/server/ToFhirLogServer.scala b/tofhir-log-server/src/main/scala/io/tofhir/log/server/ToFhirLogServer.scala deleted file mode 100644 index feaa6963..00000000 --- a/tofhir-log-server/src/main/scala/io/tofhir/log/server/ToFhirLogServer.scala +++ /dev/null @@ -1,15 +0,0 @@ -package io.tofhir.log.server - -import io.tofhir.log.server.endpoint.ExecutionEndpoint -import io.tofhir.server.common.config.WebServerConfig - -object ToFhirLogServer { - def start(): Unit = { - import io.tofhir.engine.Execution.actorSystem - - val webServerConfig = new WebServerConfig(actorSystem.settings.config.getConfig("webserver")) - val endpoint = new ExecutionEndpoint(webServerConfig) - - ToFhirLogHttpServer.start(endpoint.toFHIRRoute, webServerConfig) - } -} diff --git a/tofhir-log-server/src/main/scala/io/tofhir/log/server/config/ToFhirLogServerConfig.scala b/tofhir-log-server/src/main/scala/io/tofhir/log/server/config/ToFhirLogServerConfig.scala deleted file mode 100644 index 2b79ee05..00000000 --- a/tofhir-log-server/src/main/scala/io/tofhir/log/server/config/ToFhirLogServerConfig.scala +++ /dev/null @@ -1,24 +0,0 @@ -package io.tofhir.log.server.config - -import com.typesafe.config.Config - -/** - * ToFhir log-server configurations - */ -object ToFhirLogServerConfig { - - /** - * Get config file - */ - import io.tofhir.engine.Execution.actorSystem - protected lazy val config: Config = actorSystem.settings.config - - /** - * Config for toFhir log server - */ - private lazy val toFhirLogServerConfig: Config = config.getConfig("tofhir.log-server") - - /** Path of the file that contains results of mapping executions */ - lazy val mappingLogsFilePath: String = toFhirLogServerConfig.getString("filepath") - -} diff --git a/tofhir-log-server/src/main/scala/io/tofhir/log/server/endpoint/ExecutionEndpoint.scala b/tofhir-log-server/src/main/scala/io/tofhir/log/server/endpoint/ExecutionEndpoint.scala deleted file mode 100644 index 666d03a0..00000000 --- a/tofhir-log-server/src/main/scala/io/tofhir/log/server/endpoint/ExecutionEndpoint.scala +++ /dev/null @@ -1,98 +0,0 @@ -package io.tofhir.log.server.endpoint - -import akka.http.scaladsl.model.headers.RawHeader -import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.Route -import io.tofhir.common.model.Json4sSupport._ -import com.typesafe.scalalogging.LazyLogging -import io.tofhir.server.common.interceptor.IErrorHandler -import io.tofhir.log.server.service.ExecutionService -import ExecutionEndpoint.{SEGMENT_EXECUTIONS, SEGMENT_JOB, SEGMENT_LOGS, SEGMENT_PROJECTS} -import io.tofhir.server.common.config.WebServerConfig -import io.tofhir.server.common.interceptor.ICORSHandler - -class ExecutionEndpoint(webServerConfig: WebServerConfig) extends ICORSHandler with IErrorHandler with LazyLogging { - - val executionService: ExecutionService = new ExecutionService() - - lazy val toFHIRRoute: Route = - pathPrefix(webServerConfig.baseUri) { - corsHandler { - pathPrefix(SEGMENT_PROJECTS) { - pathPrefix(Segment) { projectId: String => { - pathPrefix(SEGMENT_JOB) { - pathPrefix(Segment) { jobId: String => - pathPrefix(SEGMENT_EXECUTIONS) { - pathEndOrSingleSlash { - getExecutions(projectId, jobId) // Get all executions for this job, jobs//executions - } ~ pathPrefix(Segment) { executionId: String => - pathEndOrSingleSlash { - getExecutionById(projectId, jobId, executionId) // get an execution, jobs//executions/ - } ~ pathPrefix(SEGMENT_LOGS) { - pathEndOrSingleSlash { - getExecutionLogs(executionId) // get logs of an execution, jobs//executions//logs - } - } - } - } - } - } - } - } - } - } - } - - /** - * Route to get executions of a mapping job - * */ - private def getExecutions(projectId: String, id: String): Route = { - get { - parameterMap { queryParams => // page and filter information is included (Ex: page=1&errorStatus=SUCCESS,FAILURE) - onComplete(executionService.getExecutions(projectId, id, queryParams)) { - case util.Success(response) => - val headers = List( - RawHeader(ICORSHandler.X_TOTAL_COUNT_HEADER, response._2.toString) - ) - respondWithHeaders(headers) { - complete(response._1) - } - } - } - } - } - - /** - * Route to get execution logs of a mapping job execution - * - * @param projectId - * @param jobId - * @param executionId - * @return - */ - private def getExecutionById(projectId: String, jobId: String, executionId: String): Route = { - get { - complete { - executionService.getExecutionById(projectId, jobId, executionId) - } - } - } - - /** - * Route to retrieve execution logs i.e. the logs of mapping task which are ran in the execution - * */ - private def getExecutionLogs(id: String): Route = { - get { - complete { - executionService.getExecutionLogs(id) - } - } - } -} - -object ExecutionEndpoint { - val SEGMENT_PROJECTS = "projects" - val SEGMENT_JOB = "jobs" - val SEGMENT_EXECUTIONS = "executions" - val SEGMENT_LOGS = "logs" -} diff --git a/tofhir-log-server/src/main/scala/io/tofhir/log/server/service/ExecutionService.scala b/tofhir-log-server/src/main/scala/io/tofhir/log/server/service/ExecutionService.scala deleted file mode 100644 index d7791f23..00000000 --- a/tofhir-log-server/src/main/scala/io/tofhir/log/server/service/ExecutionService.scala +++ /dev/null @@ -1,266 +0,0 @@ -package io.tofhir.log.server.service - -import com.typesafe.scalalogging.LazyLogging -import io.tofhir.engine.config.ToFhirConfig -import io.tofhir.log.server.config.ToFhirLogServerConfig -import io.tofhir.server.common.model.ResourceNotFound -import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder -import org.apache.spark.sql.functions.col -import org.apache.spark.sql.types._ -import org.apache.spark.sql.{Encoders, Row} -import org.json4s.JsonAST.{JObject, JValue} -import org.json4s.jackson.JsonMethods -import org.json4s.{JArray, JString} - -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future - -/** - * Service to handle all execution related operations - * E.g. Run a mapping job, run a mapping task, run a test resource creation, get execution logs - * - */ -class ExecutionService() extends LazyLogging { - - /** - * Returns the logs of mapping tasks ran in the given execution. - * - * @param executionId the identifier of mapping job execution. - * @return the logs of mapping tasks - * */ - def getExecutionLogs(executionId: String): Future[Seq[JValue]] = { - Future { - // read the logs file - val dataFrame = ToFhirConfig.sparkSession.read.json(ToFhirLogServerConfig.mappingLogsFilePath) - // handle the case where no job has been run yet which makes the data frame empty - if (dataFrame.isEmpty) { - Seq.empty - } - else { - // Get mapping tasks logs for the given execution. ProjectId field is not null for selecting mappingTasksLogs, filter out row error logs. - val mappingTasksLogs = dataFrame.filter(s"executionId = '$executionId' and projectId is not null") - - // Handle the case where the job has not been run yet, which makes the data frame empty - if (mappingTasksLogs.isEmpty) { - Seq.empty - } else { - // Collect mapping tasks logs for matching with mappingUrl field of row error logs - var mappingTasksLogsData = mappingTasksLogs.collect() - - // Get row error logs for the given execution. ProjectId field is null for selecting row error logs, filter out mappingTasksLogs. - var rowErrorLogs = dataFrame.filter(s"executionId = '$executionId' and projectId is null") - - // Check whether there is any row error - if (!rowErrorLogs.isEmpty) { - - // Select needed columns from row error logs - rowErrorLogs = rowErrorLogs.select(List("errorCode", "errorDesc", "message", "mappingUrl").map(col): _*) - - // Group row error logs by mapping url - val rowErrorLogsGroupedByMappingUrl = rowErrorLogs.groupByKey(row => row.get(row.fieldIndex("mappingUrl")).toString)(Encoders.STRING) - - // Add row error details to mapping tasks logs if any error occurred while executing the mapping task. - val mappingTasksErrorLogsWithRowErrorLogs = rowErrorLogsGroupedByMappingUrl.mapGroups((mappingUrl, rowError) => { - // Find the related mapping task log to given mapping url - val mappingTaskLog = mappingTasksLogsData.filter(row => row.getAs[String]("mappingUrl") == mappingUrl) - // Append row error logs to the related mapping task log - Row.fromSeq(Row.unapplySeq(mappingTaskLog.head).get :+ rowError.toSeq) - })( - // Define a new schema for the resulting rows and create an encoder for it. We will add a "error_logs" column to mapping tasks logs that contains related error logs. - ExpressionEncoder(mappingTasksLogs.schema.add("error_logs", ArrayType( - new StructType() - .add("errorCode", StringType) - .add("errorDesc", StringType) - .add("message", StringType) - .add("mappingUrl", StringType) - ))) - ) - - // Build a map for updated mapping tasks logs (mappingUrl -> mapping logs with errors) - val updatedMappingTasksLogsMap = mappingTasksErrorLogsWithRowErrorLogs.collect().map(mappingLogsWithErrors => - (mappingLogsWithErrors.getAs[String]("mappingUrl"), mappingLogsWithErrors.getAs[String]("@timestamp")) -> mappingLogsWithErrors - ).toMap - - // Replace mapping task logs if it is in the map - mappingTasksLogsData = mappingTasksLogsData.map(mappingTaskLog => - updatedMappingTasksLogsMap.getOrElse((mappingTaskLog.getAs[String]("mappingUrl"), mappingTaskLog.getAs[String]("@timestamp")), mappingTaskLog)) - - } - - // return json objects for mapping tasks logs - mappingTasksLogsData.map(row => { - JsonMethods.parse(row.json).asInstanceOf[JObject] - }) - } - } - } - } - - /** - * Returns the list of mapping job executions. It extracts the logs from {@link logs/ tofhir - mappings.log} file for - * the given mapping job and groups them by their execution id and returns a single log for each execution. Further, - * it applies the pagination to the resulting execution logs. - * - * @param projectId project id the job belongs to - * @param jobId job id - * @param queryParams parameters to filter results such as paging - * @return a tuple as follows - * first element is the execution logs of mapping job as a JSON array. It returns an empty array if the job has not been run before. - * second element is the total number of executions without applying any filters i.e. query params - * @throws ResourceNotFound when mapping job does not exist - */ - def getExecutions(projectId: String, jobId: String, queryParams: Map[String, String]): Future[(Seq[JValue], Long)] = { - // retrieve the job to validate its existence - - Future { - // read the logs file - val dataFrame = ToFhirConfig.sparkSession.read.json(ToFhirLogServerConfig.mappingLogsFilePath) - // handle the case where no job has been run yet which makes the data frame empty - if (dataFrame.isEmpty) { - (Seq.empty, 0) - } - else { - val jobRuns = dataFrame.filter(s"jobId = '$jobId' and projectId = '$projectId'") - // handle the case where the job has not been run yet which makes the data frame empty - if (jobRuns.isEmpty) { - (Seq.empty, 0) - } else { - // group logs by execution id - val jobRunsGroupedByExecutionId = jobRuns.groupByKey(row => row.get(row.fieldIndex("executionId")).toString)(Encoders.STRING) - // get execution logs - val executionLogs = jobRunsGroupedByExecutionId.mapGroups((key, values) => { - // keeps the rows belonging to this execution - val rows: Seq[Row] = values.toSeq - // Extract values from the "mappingUrl" column using direct attribute access - val mappingUrls = rows.map(_.getAs[String]("mappingUrl")).distinct - // use the timestamp of first one, which is ran first, as timestamp of execution - val timestamp = rows.head.get(rows.head.fieldIndex("@timestamp")).toString - // Check if there is a row with result other than STARTED - val results: Seq[String] = rows.map(row => row.get(row.fieldIndex("result")).toString) - val status: String = ExecutionService.getErrorStatusOfExecution(results) - Row.fromSeq(Seq(key, mappingUrls, timestamp, status)) - })(ExpressionEncoder(StructType( - StructField("id", StringType) :: - StructField("mappingUrls", ArrayType(StringType)) :: - StructField("startTime", StringType) :: - StructField("errorStatus", StringType) :: Nil - ))) - // get the filter parameters - val dateBefore = queryParams.getOrElse("dateBefore", "") - val dateAfter = queryParams.getOrElse("dateAfter", "") - val errorStatuses = queryParams.getOrElse("errorStatuses", "") - - var filteredLogs = executionLogs - // Filter according to error status of the execution - if (errorStatuses.nonEmpty) { - filteredLogs = filteredLogs.filter(col("errorStatus").isin(errorStatuses.split(","): _*)) - } - - // Filter according to start date - if (dateAfter.nonEmpty) { - filteredLogs = filteredLogs.filter(col("startTime") > dateAfter) - } - if (dateBefore.nonEmpty) { - filteredLogs = filteredLogs.filter(col("startTime") < dateBefore) - } - // get length before doing pagination - val filteredCount = filteredLogs.count() - - val pageSize: Int = queryParams.getOrElse("rowsPerPage", "10").toInt - // handle the pagination according to the page size - val numOfPages = Math.ceil(filteredCount.toDouble / pageSize).toInt - val page = queryParams.getOrElse("page", "1").toInt - - if (page > numOfPages) { - (Seq.empty, 0) - } else { - // handle the case where requested page does not exist - val start = (page - 1) * pageSize - val end = Math.min(start + pageSize, filteredCount.toInt) - - // sort the executions by latest to oldest - val paginatedLogs = filteredLogs.sort(filteredLogs.col("startTime").desc).collect().slice(start, end) - - // Retrieve the running executions for the given job - (paginatedLogs.map(row => { - JsonMethods.parse(row.json).asInstanceOf[JObject] - }), filteredCount) - } - } - } - } - } - - /** - * Returns the execution logs for a specific execution ID. - * - * @param projectId project id the job belongs to - * @param jobId job id - * @param executionId execution id - * @return the execution summary as a JSON object - */ - def getExecutionById(projectId: String, jobId: String, executionId: String): Future[JObject] = { - // Retrieve the job to validate its existence - Future { - // Read the logs file - val dataFrame = ToFhirConfig.sparkSession.read.json(ToFhirLogServerConfig.mappingLogsFilePath) - // Filter logs by job and execution ID - val filteredLogs = dataFrame.filter(s"jobId = '$jobId' and projectId = '$projectId' and executionId = '$executionId'") - // Check if any logs exist for the given execution - if (filteredLogs.isEmpty) { // execution not found, return response with 404 status code - throw ResourceNotFound("Execution does not exists.", s"An execution with id $executionId does not exists.") - } else { - // Extract values from the "mappingUrl" column using direct attribute access - val mappingUrls = filteredLogs.select("mappingUrl").distinct().collect().map(_.getString(0)).toList - // Use the timestamp of the first log as the execution timestamp - val timestamp = filteredLogs.select("@timestamp").first().getString(0) - // Determine the status based on the success count - val results: Seq[String] = filteredLogs.select("result").collect().map(_.getString(0)).toSeq - val status: String = ExecutionService.getErrorStatusOfExecution(results) - // Create a JSON object representing the execution - val executionJson = JObject( - "id" -> JString(executionId), - "mappingUrls" -> JArray(mappingUrls.map(JString)), - "startTime" -> JString(timestamp), - "errorStatus" -> JString(status) - ) - executionJson - } - } - } - -} - -object ExecutionService { - - /** - * Determines the error status of the execution based on the results of the mapping tasks. - * - * @param results - * @return - */ - private def getErrorStatusOfExecution(results: Seq[String]): String = { - if (results.exists(_ == "PARTIAL_SUCCESS")) { - "PARTIAL_SUCCESS" - } else { - // success > 0 and failure = 0 means success - // success > 0 and failure > 0 means partial success - // success = 0 and failure > 0 means failure - // success = 0 and failure = 0 means started - val successCount = results.count(_ == "SUCCESS") - val failureCount = results.count(_ == "FAILURE") - if (successCount > 0 && failureCount == 0) { - "SUCCESS" - } else if (successCount > 0 && failureCount > 0) { - "PARTIAL_SUCCESS" - } else if (failureCount > 0) { - "FAILURE" - } else { - "STARTED" - } - } - } -} - - - diff --git a/tofhir-log-server/src/test/resources/application.conf b/tofhir-log-server/src/test/resources/application.conf deleted file mode 100644 index 5d446db9..00000000 --- a/tofhir-log-server/src/test/resources/application.conf +++ /dev/null @@ -1,6 +0,0 @@ -tofhir = { - log-server { - # The file that contains results of mapping executions - filepath = "tofhir-log-server/src/test/resources/log-sample.log" - } -} \ No newline at end of file diff --git a/tofhir-log-server/src/test/resources/log-sample.log b/tofhir-log-server/src/test/resources/log-sample.log deleted file mode 100644 index 85c71c85..00000000 --- a/tofhir-log-server/src/test/resources/log-sample.log +++ /dev/null @@ -1,14 +0,0 @@ -{"@timestamp":"2023-12-25T17:19:06.971+03:00","message":"toFHIR batch mapping result (STARTED) for execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14' of job 'pilot1-preop' in project 'pilot1' for mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping'!\n\t# of Invalid Rows: \t-1\n\t# of Not Mapped: \t-1\n\t# of Failed writes:\t-1\n\t# of Written FHIR resources:\t-1","logger_name":"io.tofhir.engine.mapping.FhirMappingJobManager","level":"INFO","jobId":"pilot1-preop","projectId":"pilot1","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","result":"STARTED","numOfInvalids":-1,"numOfNotMapped":-1,"numOfFhirResources":-1,"numOfFailedWrites":-1,"eventId":"MAPPING_JOB_RESULT"} -{"@timestamp":"2023-12-25T17:19:15.005+03:00","message":"toFHIR batch mapping result (FAILURE) for execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14' of job 'pilot1-preop' in project 'pilot1' for mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping'!\n\t# of Invalid Rows: \t0\n\t# of Not Mapped: \t0\n\t# of Failed writes:\t10\n\t# of Written FHIR resources:\t0","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"INFO","jobId":"pilot1-preop","projectId":"pilot1","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","result":"FAILURE","numOfInvalids":0,"numOfNotMapped":0,"numOfFhirResources":0,"numOfFailedWrites":10,"eventId":"MAPPING_JOB_RESULT"} -{"@timestamp":"2023-12-25T17:19:15.006+03:00","message":"Mapping failure (service_error) for job 'pilot1-preop' and mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping' within expression 'result' execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14'!\n\tSource: {\"pid\":\"p1\",\"gender\":\"male\",\"birthDate\":\"2000-05-10\",\"deceasedDateTime\":null,\"homePostalCode\":null}\n\tError: FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"WARN","jobId":"pilot1-preop","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","mappingExpr":"result","source":{"pid":"p1","gender":"male","birthDate":"2000-05-10","deceasedDateTime":null,"homePostalCode":null},"errorCode":"service_error","errorDesc":"FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","errorExpr":null,"eventId":"MAPPING_RESULT"} -{"@timestamp":"2023-12-25T17:19:15.006+03:00","message":"Mapping failure (service_error) for job 'pilot1-preop' and mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping' within expression 'result' execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14'!\n\tSource: {\"pid\":\"p2\",\"gender\":\"male\",\"birthDate\":\"1985-05-08\",\"deceasedDateTime\":\"2017-03-10\",\"homePostalCode\":\"G02547\"}\n\tError: FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"WARN","jobId":"pilot1-preop","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","mappingExpr":"result","source":{"pid":"p2","gender":"male","birthDate":"1985-05-08","deceasedDateTime":"2017-03-10","homePostalCode":"G02547"},"errorCode":"service_error","errorDesc":"FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","errorExpr":null,"eventId":"MAPPING_RESULT"} -{"@timestamp":"2023-12-25T17:19:15.006+03:00","message":"Mapping failure (service_error) for job 'pilot1-preop' and mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping' within expression 'result' execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14'!\n\tSource: {\"pid\":\"p3\",\"gender\":\"male\",\"birthDate\":\"1997-02\",\"deceasedDateTime\":null,\"homePostalCode\":null}\n\tError: FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"WARN","jobId":"pilot1-preop","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","mappingExpr":"result","source":{"pid":"p3","gender":"male","birthDate":"1997-02","deceasedDateTime":null,"homePostalCode":null},"errorCode":"service_error","errorDesc":"FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","errorExpr":null,"eventId":"MAPPING_RESULT"} -{"@timestamp":"2023-12-25T17:19:15.006+03:00","message":"Mapping failure (service_error) for job 'pilot1-preop' and mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping' within expression 'result' execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14'!\n\tSource: {\"pid\":\"p4\",\"gender\":\"male\",\"birthDate\":\"1999-06-05\",\"deceasedDateTime\":null,\"homePostalCode\":\"H10564\"}\n\tError: FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"WARN","jobId":"pilot1-preop","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","mappingExpr":"result","source":{"pid":"p4","gender":"male","birthDate":"1999-06-05","deceasedDateTime":null,"homePostalCode":"H10564"},"errorCode":"service_error","errorDesc":"FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","errorExpr":null,"eventId":"MAPPING_RESULT"} -{"@timestamp":"2023-12-25T17:19:15.006+03:00","message":"Mapping failure (service_error) for job 'pilot1-preop' and mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping' within expression 'result' execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14'!\n\tSource: {\"pid\":\"p5\",\"gender\":\"male\",\"birthDate\":\"1965-10-01\",\"deceasedDateTime\":\"2019-04-21\",\"homePostalCode\":\"G02547\"}\n\tError: FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"WARN","jobId":"pilot1-preop","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","mappingExpr":"result","source":{"pid":"p5","gender":"male","birthDate":"1965-10-01","deceasedDateTime":"2019-04-21","homePostalCode":"G02547"},"errorCode":"service_error","errorDesc":"FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","errorExpr":null,"eventId":"MAPPING_RESULT"} -{"@timestamp":"2023-12-25T17:19:15.006+03:00","message":"Mapping failure (service_error) for job 'pilot1-preop' and mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping' within expression 'result' execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14'!\n\tSource: {\"pid\":\"p6\",\"gender\":\"female\",\"birthDate\":\"1991-03\",\"deceasedDateTime\":null,\"homePostalCode\":null}\n\tError: FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"WARN","jobId":"pilot1-preop","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","mappingExpr":"result","source":{"pid":"p6","gender":"female","birthDate":"1991-03","deceasedDateTime":null,"homePostalCode":null},"errorCode":"service_error","errorDesc":"FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","errorExpr":null,"eventId":"MAPPING_RESULT"} -{"@timestamp":"2023-12-25T17:19:15.006+03:00","message":"Mapping failure (service_error) for job 'pilot1-preop' and mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping' within expression 'result' execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14'!\n\tSource: {\"pid\":\"p7\",\"gender\":\"female\",\"birthDate\":\"1972-10-25\",\"deceasedDateTime\":null,\"homePostalCode\":\"V13135\"}\n\tError: FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"WARN","jobId":"pilot1-preop","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","mappingExpr":"result","source":{"pid":"p7","gender":"female","birthDate":"1972-10-25","deceasedDateTime":null,"homePostalCode":"V13135"},"errorCode":"service_error","errorDesc":"FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","errorExpr":null,"eventId":"MAPPING_RESULT"} -{"@timestamp":"2023-12-25T17:19:15.007+03:00","message":"Mapping failure (service_error) for job 'pilot1-preop' and mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping' within expression 'result' execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14'!\n\tSource: {\"pid\":\"p8\",\"gender\":\"female\",\"birthDate\":\"2010-01-10\",\"deceasedDateTime\":null,\"homePostalCode\":\"Z54564\"}\n\tError: FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"WARN","jobId":"pilot1-preop","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","mappingExpr":"result","source":{"pid":"p8","gender":"female","birthDate":"2010-01-10","deceasedDateTime":null,"homePostalCode":"Z54564"},"errorCode":"service_error","errorDesc":"FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","errorExpr":null,"eventId":"MAPPING_RESULT"} -{"@timestamp":"2023-12-25T17:19:15.007+03:00","message":"Mapping failure (service_error) for job 'pilot1-preop' and mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping' within expression 'result' execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14'!\n\tSource: {\"pid\":\"p9\",\"gender\":\"female\",\"birthDate\":\"1999-05-12\",\"deceasedDateTime\":null,\"homePostalCode\":null}\n\tError: FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"WARN","jobId":"pilot1-preop","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","mappingExpr":"result","source":{"pid":"p9","gender":"female","birthDate":"1999-05-12","deceasedDateTime":null,"homePostalCode":null},"errorCode":"service_error","errorDesc":"FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","errorExpr":null,"eventId":"MAPPING_RESULT"} -{"@timestamp":"2023-12-25T17:19:15.007+03:00","message":"Mapping failure (service_error) for job 'pilot1-preop' and mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping' within expression 'result' execution 'b9d60a61-6d0b-4ea1-9c4e-84e343334b14'!\n\tSource: {\"pid\":\"p10\",\"gender\":\"female\",\"birthDate\":\"2003-11\",\"deceasedDateTime\":null,\"homePostalCode\":null}\n\tError: FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"WARN","jobId":"pilot1-preop","executionId":"b9d60a61-6d0b-4ea1-9c4e-84e343334b14","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","mappingExpr":"result","source":{"pid":"p10","gender":"female","birthDate":"2003-11","deceasedDateTime":null,"homePostalCode":null},"errorCode":"service_error","errorDesc":"FHIR repository at url http://localhost:8082 returned an unidentified error while writing the resources!","errorExpr":null,"eventId":"MAPPING_RESULT"} -{"@timestamp":"2023-12-26T09:18:22.408+03:00","message":"toFHIR batch mapping result (STARTED) for execution '70503f99-8aea-4b4f-8b1b-6dbc3d5509fe' of job 'pilot1-preop' in project 'pilot1' for mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping'!\n\t# of Invalid Rows: \t-1\n\t# of Not Mapped: \t-1\n\t# of Failed writes:\t-1\n\t# of Written FHIR resources:\t-1","logger_name":"io.tofhir.engine.mapping.FhirMappingJobManager","level":"INFO","jobId":"pilot1-preop","projectId":"pilot1","executionId":"70503f99-8aea-4b4f-8b1b-6dbc3d5509fe","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","result":"STARTED","numOfInvalids":-1,"numOfNotMapped":-1,"numOfFhirResources":-1,"numOfFailedWrites":-1,"eventId":"MAPPING_JOB_RESULT"} -{"@timestamp":"2023-12-26T09:18:23.72+03:00","message":"toFHIR batch mapping result (SUCCESS) for execution '70503f99-8aea-4b4f-8b1b-6dbc3d5509fe' of job 'pilot1-preop' in project 'pilot1' for mapping 'https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping'!\n\t# of Invalid Rows: \t0\n\t# of Not Mapped: \t0\n\t# of Failed writes:\t0\n\t# of Written FHIR resources:\t10","logger_name":"io.tofhir.engine.data.write.SinkHandler$","level":"INFO","jobId":"pilot1-preop","projectId":"pilot1","executionId":"70503f99-8aea-4b4f-8b1b-6dbc3d5509fe","mappingUrl":"https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping","result":"SUCCESS","numOfInvalids":0,"numOfNotMapped":0,"numOfFhirResources":10,"numOfFailedWrites":0,"eventId":"MAPPING_JOB_RESULT"} diff --git a/tofhir-log-server/src/test/scala/io/tofhir/server/service/ExecutionServiceTest.scala b/tofhir-log-server/src/test/scala/io/tofhir/server/service/ExecutionServiceTest.scala deleted file mode 100644 index eeec4baa..00000000 --- a/tofhir-log-server/src/test/scala/io/tofhir/server/service/ExecutionServiceTest.scala +++ /dev/null @@ -1,116 +0,0 @@ -package io.tofhir.server.service - -import io.tofhir.engine.Execution.actorSystem.dispatcher -import io.tofhir.engine.model.FhirMappingJobResult -import io.tofhir.common.model.Json4sSupport.formats -import io.tofhir.log.server.service.ExecutionService -import org.json4s.JsonAST.JArray -import org.scalatest.BeforeAndAfterAll -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AsyncWordSpec - -/** - * Tests for the ExecutionService. - * - * This suite focuses on testing the functionality of the ExecutionService, which works with the log file provided - * in the test/resources/log-sample.log file. The log file includes records for the following scenarios: - * - An execution of a mapping job with one mapping, which fails due to an incorrect FHIR Repository URL. - * - An execution of a mapping job with one mapping, which completes successfully. - */ -class ExecutionServiceTest extends AsyncWordSpec with Matchers with BeforeAndAfterAll { - - val executionService: ExecutionService = new ExecutionService - // The identifier for the project - private val projectId = "pilot1" - // The identifier for the specific job - private val jobId = "pilot1-preop" - // The identifier for the first mapping job execution - private val firstMappingJobExecutionId = "b9d60a61-6d0b-4ea1-9c4e-84e343334b14" - // The identifier for the second mapping job execution - private val secondMappingJobExecutionId = "70503f99-8aea-4b4f-8b1b-6dbc3d5509fe" - - "The Execution Service" should { - - "should get mapping job executions" in { - executionService.getExecutions(projectId, jobId, Map.empty) - .map(executions => { - val executionsData = executions._1 - val count = executions._2 - count shouldEqual 2 - executionsData.length shouldEqual count - (executionsData.head \ "id").extract[String] shouldEqual secondMappingJobExecutionId - (executionsData.head \ "errorStatus").extract[String] shouldEqual FhirMappingJobResult.SUCCESS - (executionsData.head \ "mappingUrls").extract[Seq[String]].length shouldEqual 1 - - (executionsData(1) \ "id").extract[String] shouldEqual firstMappingJobExecutionId - (executionsData(1) \ "errorStatus").extract[String] shouldEqual FhirMappingJobResult.FAILURE - (executionsData(1) \ "mappingUrls").extract[Seq[String]].length shouldEqual 1 - }) - } - - "should get executions filtered by a date range" in { - val queryParams = Map("dateBefore" -> "2023-12-27", "dateAfter" -> "2023-12-26") - executionService.getExecutions(projectId, jobId, queryParams) - .map(executions => { - val executionsData = executions._1 - val count = executions._2 - count shouldEqual 1 - executionsData.length shouldEqual count - (executionsData.head \ "id").extract[String] shouldEqual secondMappingJobExecutionId - (executionsData.head \ "errorStatus").extract[String] shouldEqual FhirMappingJobResult.SUCCESS - (executionsData.head \ "mappingUrls").extract[Seq[String]].length shouldEqual 1 - (executionsData.head \ "mappingUrls")(0).extract[String] shouldEqual "https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping" - }) - } - - "should not get any execution when page number is greater than the maximum" in { - val queryParams = Map("dateBefore" -> "2023-12-27", "dateAfter" -> "2023-12-03", "rowsPerPage" -> "4", "page" -> "3") - executionService.getExecutions(projectId, jobId, queryParams) - .map(executions => { - val executionsData = executions._1 - val count = executions._2 - count shouldEqual 0 - executionsData.length shouldEqual count - }) - } - - "should get executions filtered by error status" in { - val queryParams = Map("errorStatuses" -> FhirMappingJobResult.SUCCESS) - executionService.getExecutions(projectId, jobId, queryParams) - .map(executions => { - val executionsData = executions._1 - val count = executions._2 - count shouldEqual 1 - executionsData.length shouldEqual count - (executionsData.head \ "id").extract[String] shouldEqual secondMappingJobExecutionId - (executionsData.head \ "errorStatus").extract[String] shouldEqual FhirMappingJobResult.SUCCESS - (executionsData.head \ "mappingUrls").extract[Seq[String]].length shouldEqual 1 - }) - } - - "should get execution by id" in { - executionService.getExecutionById(projectId, jobId, secondMappingJobExecutionId) - .map(execution => { - (execution \ "id").extract[String] shouldEqual secondMappingJobExecutionId - (execution \ "errorStatus").extract[String] shouldEqual FhirMappingJobResult.SUCCESS - (execution \ "mappingUrls").extract[Seq[String]].length shouldEqual 1 - (execution \ "mappingUrls")(0).extract[String] shouldEqual "https://aiccelerate.eu/fhir/mappings/pilot1/patient-mapping" - }) - } - - "should get execution logs by execution id" in { - executionService.getExecutionLogs(firstMappingJobExecutionId) - .map(logs => { - logs.length shouldEqual 2 - - // check the first log -> STARTED - (logs.head \ "result").extract[String] shouldEqual FhirMappingJobResult.STARTED - (logs.head \ "error_logs").extract[JArray].arr.size shouldEqual 10 - - // check the second log -> FAILURE - (logs(1) \ "result").extract[String] shouldEqual FhirMappingJobResult.FAILURE - (logs(1) \ "numOfFailedWrites").extract[String] shouldEqual "10" - }) - } - } -} diff --git a/tofhir-server/pom.xml b/tofhir-server/pom.xml index 5758fc89..d194b2d6 100644 --- a/tofhir-server/pom.xml +++ b/tofhir-server/pom.xml @@ -183,7 +183,15 @@ mockito-scala_2.13 test - + + + com.sndyuk + logback-more-appenders + + + org.fluentd + fluent-logger + com.typesafe.akka diff --git a/tofhir-server/src/main/resources/application.conf b/tofhir-server/src/main/resources/application.conf index 3bfcfc11..b80461fe 100644 --- a/tofhir-server/src/main/resources/application.conf +++ b/tofhir-server/src/main/resources/application.conf @@ -137,11 +137,6 @@ webserver = { } } -# The service from where tofhir-server will read the logs. -log-service = { - endpoint = "http://localhost:8086/tofhir-logs" -} - # Spark configurations spark = { app.name = "AICCELERATE Data Integration Suite" diff --git a/tofhir-server/src/main/resources/logback.xml b/tofhir-server/src/main/resources/logback.xml index b1100c41..f62ebd10 100644 --- a/tofhir-server/src/main/resources/logback.xml +++ b/tofhir-server/src/main/resources/logback.xml @@ -50,6 +50,9 @@ [ignore] + + ${LOG_FOLDER}/tofhir-mappings.%i.log.zip @@ -61,11 +64,27 @@ 10MB + + + + localhost + 24224 + + + INFO + + + true + + + + + @@ -76,10 +95,12 @@ + + diff --git a/tofhir-server/src/main/scala/io/tofhir/server/ToFhirServer.scala b/tofhir-server/src/main/scala/io/tofhir/server/ToFhirServer.scala index 329e3e6a..50237835 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/ToFhirServer.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/ToFhirServer.scala @@ -1,7 +1,7 @@ package io.tofhir.server import io.tofhir.engine.config.ToFhirConfig -import io.tofhir.server.config.{LogServiceConfig, RedCapServiceConfig} +import io.tofhir.server.config.RedCapServiceConfig import io.tofhir.server.common.config.WebServerConfig import io.tofhir.server.endpoint.ToFhirServerEndpoint import io.tofhir.server.fhir.FhirDefinitionsConfig @@ -12,9 +12,8 @@ object ToFhirServer { val webServerConfig = new WebServerConfig(actorSystem.settings.config.getConfig("webserver")) val fhirDefinitionsConfig = new FhirDefinitionsConfig(actorSystem.settings.config.getConfig("fhir")) - val logServiceConfig = new LogServiceConfig(actorSystem.settings.config.getConfig("log-service")) val redCapServiceConfig = new RedCapServiceConfig(actorSystem.settings.config.getConfig("tofhir-redcap")) - val endpoint = new ToFhirServerEndpoint(ToFhirConfig.engineConfig, webServerConfig, fhirDefinitionsConfig, logServiceConfig, redCapServiceConfig) + val endpoint = new ToFhirServerEndpoint(ToFhirConfig.engineConfig, webServerConfig, fhirDefinitionsConfig, redCapServiceConfig) ToFhirHttpServer.start(endpoint.toFHIRRoute, webServerConfig) } diff --git a/tofhir-server/src/main/scala/io/tofhir/server/config/LogServiceConfig.scala b/tofhir-server/src/main/scala/io/tofhir/server/config/LogServiceConfig.scala deleted file mode 100644 index 5c736467..00000000 --- a/tofhir-server/src/main/scala/io/tofhir/server/config/LogServiceConfig.scala +++ /dev/null @@ -1,10 +0,0 @@ -package io.tofhir.server.config - -import com.typesafe.config.Config - -import scala.util.Try - -class LogServiceConfig(logServiceConfig: Config) { - /** Host name/address to start service on. */ - lazy val logServiceEndpoint: String = Try(logServiceConfig.getString("endpoint")).getOrElse("http://localhost:8086/tofhir-log") -} diff --git a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/JobEndpoint.scala b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/JobEndpoint.scala index 8e5ae9b0..4b6a4c20 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/JobEndpoint.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/JobEndpoint.scala @@ -1,19 +1,17 @@ package io.tofhir.server.endpoint import akka.http.scaladsl.model.{HttpEntity, HttpResponse, StatusCodes} -import akka.http.scaladsl.model.headers.RawHeader import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route import akka.stream.StreamTcpException import com.typesafe.scalalogging.LazyLogging import io.tofhir.engine.model.FhirMappingJob -import io.tofhir.server.endpoint.JobEndpoint.{SEGMENT_EXECUTIONS, SEGMENT_JOB, SEGMENT_LOGS, SEGMENT_MAPPINGS, SEGMENT_RUN, SEGMENT_STATUS, SEGMENT_STOP, SEGMENT_TEST, SEGMENT_DESCHEDULE} +import io.tofhir.server.endpoint.JobEndpoint.{SEGMENT_EXECUTIONS, SEGMENT_JOB, SEGMENT_MAPPINGS, SEGMENT_RUN, SEGMENT_STATUS, SEGMENT_STOP, SEGMENT_TEST, SEGMENT_DESCHEDULE} import io.tofhir.common.model.Json4sSupport._ import io.tofhir.server.model.{ExecuteJobTask, RowSelectionOrder, TestResourceCreationRequest} import io.tofhir.server.service.{ExecutionService, JobService} import io.tofhir.engine.Execution.actorSystem.dispatcher import io.tofhir.engine.util.FhirMappingJobFormatter.formats -import io.tofhir.server.common.interceptor.ICORSHandler import io.tofhir.server.common.model.{ResourceNotFound, ToFhirRestCall} import io.tofhir.server.service.job.IJobRepository import io.tofhir.server.service.mapping.IMappingRepository @@ -21,10 +19,10 @@ import io.tofhir.server.service.schema.ISchemaRepository import scala.concurrent.Future -class JobEndpoint(jobRepository: IJobRepository, mappingRepository: IMappingRepository, schemaRepository: ISchemaRepository, logServiceEndpoint: String) extends LazyLogging { +class JobEndpoint(jobRepository: IJobRepository, mappingRepository: IMappingRepository, schemaRepository: ISchemaRepository) extends LazyLogging { val service: JobService = new JobService(jobRepository) - val executionService: ExecutionService = new ExecutionService(jobRepository, mappingRepository, schemaRepository, logServiceEndpoint) + val executionService: ExecutionService = new ExecutionService(jobRepository, mappingRepository, schemaRepository) def route(request: ToFhirRestCall): Route = { pathPrefix(SEGMENT_JOB) { @@ -50,13 +48,7 @@ class JobEndpoint(jobRepository: IJobRepository, mappingRepository: IMappingRepo pathEndOrSingleSlash { getExecutions(projectId, jobId) ~ stopExecutions(jobId) } ~ pathPrefix(Segment) { executionId: String => // operations on a single execution, jobs//executions/ - pathEndOrSingleSlash { - getExecutionById(projectId, jobId, executionId) - } ~ pathPrefix(SEGMENT_LOGS) { // logs on a single execution, jobs//executions//logs - pathEndOrSingleSlash { - getExecutionLogs(projectId, jobId, executionId) - } - } ~ pathPrefix(SEGMENT_RUN) { // jobs//executions//run + pathPrefix(SEGMENT_RUN) { // jobs//executions//run pathEndOrSingleSlash { continueJobExecution(projectId, jobId, executionId) } @@ -200,28 +192,8 @@ class JobEndpoint(jobRepository: IJobRepository, mappingRepository: IMappingRepo * */ private def getExecutions(projectId: String, id: String): Route = { get { - parameterMap { queryParams => // page and some filter options available. (page, dateBefore, dateAfter, errorStatuses, rowPerPage) - onComplete(executionService.getExecutions(projectId, id, queryParams)) { - case util.Success(response) => - val headers = List( - RawHeader(ICORSHandler.X_TOTAL_COUNT_HEADER, response._2.toString) - ) - respondWithHeaders(headers) { - complete(response._1) - } - case util.Failure(exception) => - exception match { - case e:StreamTcpException => - logger.error(s"Failed to retrieve executions for project $projectId job $id",e) - complete { - HttpResponse( - status = StatusCodes.GatewayTimeout, - entity = "The toFHIR Log Server is currently unavailable. Please try again later." - ) - } - case t:Throwable => throw t - } - } + complete { + executionService.getExecutions(projectId, id) } } } @@ -239,21 +211,6 @@ class JobEndpoint(jobRepository: IJobRepository, mappingRepository: IMappingRepo } } - /** - * Route to get execution logs of a mapping job execution - * @param projectId - * @param jobId - * @param executionId - * @return - */ - private def getExecutionById(projectId: String, jobId: String, executionId: String): Route = { - get { - complete { - executionService.getExecutionById(projectId, jobId, executionId) - } - } - } - /** * Route to continue a job execution with parameters (e.g. clearCheckpoint) * @param projectId @@ -316,17 +273,6 @@ class JobEndpoint(jobRepository: IJobRepository, mappingRepository: IMappingRepo } } } - - /** - * Route to retrieve execution logs i.e. the logs of mapping task which are ran in the execution - * */ - private def getExecutionLogs(projectId: String, jobId: String, executionId: String): Route = { - get { - complete { - executionService.getExecutionLogs(projectId: String, jobId: String, executionId: String) - } - } - } } object JobEndpoint { @@ -334,7 +280,6 @@ object JobEndpoint { val SEGMENT_RUN = "run" val SEGMENT_STATUS = "status" val SEGMENT_EXECUTIONS = "executions" - val SEGMENT_LOGS = "logs" val SEGMENT_TEST = "test" val SEGMENT_STOP = "stop" val SEGMENT_DESCHEDULE = "deschedule" diff --git a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ProjectEndpoint.scala b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ProjectEndpoint.scala index 55235588..7f7f6a15 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ProjectEndpoint.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ProjectEndpoint.scala @@ -26,13 +26,12 @@ class ProjectEndpoint(schemaRepository: ISchemaRepository, mappingRepository: IMappingRepository, jobRepository: IJobRepository, mappingContextRepository: IMappingContextRepository, - projectRepository: IProjectRepository, - logServiceEndpoint: String) extends LazyLogging { + projectRepository: IProjectRepository) extends LazyLogging { val service: ProjectService = new ProjectService(projectRepository, jobRepository, mappingRepository, mappingContextRepository, schemaRepository) val schemaDefinitionEndpoint: SchemaDefinitionEndpoint = new SchemaDefinitionEndpoint(schemaRepository, mappingRepository) val mappingEndpoint: MappingEndpoint = new MappingEndpoint(mappingRepository, jobRepository) - val jobEndpoint: JobEndpoint = new JobEndpoint(jobRepository, mappingRepository, schemaRepository, logServiceEndpoint) + val jobEndpoint: JobEndpoint = new JobEndpoint(jobRepository, mappingRepository, schemaRepository) val mappingContextEndpoint: MappingContextEndpoint = new MappingContextEndpoint(mappingContextRepository) def route(request: ToFhirRestCall): Route = { diff --git a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ToFhirServerEndpoint.scala b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ToFhirServerEndpoint.scala index e9f10340..ef6a4b75 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ToFhirServerEndpoint.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/endpoint/ToFhirServerEndpoint.scala @@ -4,7 +4,7 @@ import akka.http.scaladsl.model.{HttpMethod, Uri} import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.{RejectionHandler, Route} import io.tofhir.engine.config.ToFhirEngineConfig -import io.tofhir.server.config.{LogServiceConfig, RedCapServiceConfig} +import io.tofhir.server.config.RedCapServiceConfig import io.tofhir.server.common.config.WebServerConfig import io.tofhir.server.fhir.FhirDefinitionsConfig import io.tofhir.server.common.interceptor.{ICORSHandler, IErrorHandler} @@ -25,7 +25,7 @@ import java.util.UUID * Encapsulates all services and directives * Main Endpoint for toFHIR server */ -class ToFhirServerEndpoint(toFhirEngineConfig: ToFhirEngineConfig, webServerConfig: WebServerConfig, fhirDefinitionsConfig: FhirDefinitionsConfig, logServiceConfig: LogServiceConfig, redCapServiceConfig: RedCapServiceConfig) extends ICORSHandler with IErrorHandler { +class ToFhirServerEndpoint(toFhirEngineConfig: ToFhirEngineConfig, webServerConfig: WebServerConfig, fhirDefinitionsConfig: FhirDefinitionsConfig, redCapServiceConfig: RedCapServiceConfig) extends ICORSHandler with IErrorHandler { val projectRepository: ProjectFolderRepository = new ProjectFolderRepository(toFhirEngineConfig) // creating the repository instance globally as weed a singleton instance val mappingRepository: ProjectMappingFolderRepository = new ProjectMappingFolderRepository(toFhirEngineConfig.mappingRepositoryFolderPath, projectRepository) @@ -39,7 +39,7 @@ class ToFhirServerEndpoint(toFhirEngineConfig: ToFhirEngineConfig, webServerConf // Initialize the projects by reading the resources available in the file system new FolderDBInitializer(schemaRepository, mappingRepository, mappingJobRepository, projectRepository, mappingContextRepository).init() - val projectEndpoint = new ProjectEndpoint(schemaRepository, mappingRepository, mappingJobRepository, mappingContextRepository, projectRepository, logServiceConfig.logServiceEndpoint) + val projectEndpoint = new ProjectEndpoint(schemaRepository, mappingRepository, mappingJobRepository, mappingContextRepository, projectRepository) val fhirDefinitionsEndpoint = new FhirDefinitionsEndpoint(fhirDefinitionsConfig) val fhirPathFunctionsEndpoint = new FhirPathFunctionsEndpoint() val redcapEndpoint = new RedCapEndpoint(redCapServiceConfig) diff --git a/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala b/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala index a0c10efc..6a75c01d 100644 --- a/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala +++ b/tofhir-server/src/main/scala/io/tofhir/server/service/ExecutionService.scala @@ -17,11 +17,10 @@ import io.tofhir.server.model.{ExecuteJobTask, TestResourceCreationRequest} import io.tofhir.server.service.job.IJobRepository import io.tofhir.server.service.mapping.IMappingRepository import io.tofhir.server.service.schema.ISchemaRepository -import io.tofhir.server.util.{DataFrameUtil, LogServiceClient} +import io.tofhir.server.util.DataFrameUtil import io.tofhir.engine.mapping.MappingJobScheduler import org.apache.commons.io import org.json4s.JsonAST.{JBool, JObject, JValue} -import org.json4s.JsonDSL.jobject2assoc import org.json4s.jackson.JsonMethods import org.json4s.{JArray, JString} @@ -36,9 +35,8 @@ import scala.concurrent.{ExecutionContext, Future} * @param jobRepository * @param mappingRepository * @param schemaRepository - * @param logServiceEndpoint */ -class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappingRepository, schemaRepository: ISchemaRepository, logServiceEndpoint: String) extends LazyLogging { +class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappingRepository, schemaRepository: ISchemaRepository) extends LazyLogging { val externalMappingFunctions: Map[String, IFhirPathFunctionLibraryFactory] = Map( "rxn" -> new RxNormApiFunctionLibraryFactory("https://rxnav.nlm.nih.gov", 2), @@ -46,7 +44,6 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin ) // TODO do not define engine and client as a global variable inside the class. (Testing becomes impossible) val toFhirEngine = new ToFhirEngine(Some(mappingRepository), Some(schemaRepository), externalMappingFunctions) - val logServiceClient = new LogServiceClient(logServiceEndpoint) import Execution.actorSystem implicit val ec: ExecutionContext = actorSystem.dispatcher @@ -237,94 +234,37 @@ class ExecutionService(jobRepository: IJobRepository, mappingRepository: IMappin } /** - * Returns the logs of mapping tasks ran in the given execution. - * - * @param executionId the identifier of mapping job execution. - * @return the logs of mapping tasks - * */ - def getExecutionLogs(projectId: String, jobId: String, executionId: String): Future[Seq[JValue]] = { - logServiceClient.getExecutionLogs(projectId, jobId, executionId) - .map(mappingTasksLogsResponse => { - logger.debug(s"Retrieved execution logs for projectId: $projectId, jobId: $jobId, executionId: $executionId") - mappingTasksLogsResponse.map(logResponse => { - val logResponseObject: JObject = logResponse.asInstanceOf[JObject] - - JObject( - logResponseObject.obj :+ ("runningStatus" -> JBool( - toFhirEngine.runningJobRegistry.executionExists((logResponseObject \ "jobId").extract[String], executionId, (logResponseObject \ "mappingUrl").extractOpt[String]) - ))) - }) - }) - } - - /** - * Returns the list of mapping job executions. It extracts the logs from {@link logs/ tofhir - mappings.log} file for - * the given mapping job and groups them by their execution id and returns a single log for each execution. Further, - * it applies the pagination to the resulting execution logs. + * Returns the ongoing (i.e. running or scheduled) mapping job executions. * * @param projectId project id the job belongs to * @param jobId job id - * @param queryParams parameters to filter results such as paging - * @return a tuple as follows - * first element is the execution logs of mapping job as a JSON array. It returns an empty array if the job has not been run before. - * second element is the total number of executions without applying any filters i.e. query params + * @return a list of JSONs indicating the execution id and its status i.e. runningStatus or scheduled * @throws ResourceNotFound when mapping job does not exist */ - def getExecutions(projectId: String, jobId: String, queryParams: Map[String, String]): Future[(Seq[JValue], Long)] = { + def getExecutions(projectId: String, jobId: String): Future[Seq[JValue]] = { // retrieve the job to validate its existence jobRepository.getJob(projectId, jobId).flatMap { case Some(_) => - // Desired page number - val page = queryParams.getOrElse("page", "1") - // Request executions before this date - val dateBefore = queryParams.getOrElse("dateBefore", "") - // Request executions after this date - val dateAfter = queryParams.getOrElse("dateAfter", "") - // Desired error statuses - val errorStatuses = queryParams.getOrElse("errorStatuses", "") - // log count per page - val rowPerPage = queryParams.getOrElse("rowPerPage", "10") - - logServiceClient.getExecutions(projectId, jobId, page, rowPerPage, dateBefore, dateAfter, errorStatuses).map(paginatedLogsResponse => { - logger.debug(s"Retrieved executions for projectId: $projectId, jobId: $jobId, page: $page") - // Retrieve the running executions for the given job - val jobExecutions: Set[String] = toFhirEngine.runningJobRegistry.getRunningExecutions(jobId) - - val ret = paginatedLogsResponse._1.map(log => { - val logJson: JObject = log.asInstanceOf[JObject] - val executionId: String = (logJson \ "id").extract[String] - JObject( - logJson.obj :+ ("runningStatus" -> JBool(jobExecutions.contains(executionId))) :+ - ("scheduled" -> JBool(toFhirEngine.runningJobRegistry.isScheduled(jobId, executionId))) + // Retrieve the running executions for the given job + val runningExecutionsJson: Seq[JValue] = toFhirEngine.runningJobRegistry.getRunningExecutions(jobId) + .map(id => JObject( + List( + "id" -> JString(id), + "runningStatus" -> JBool(true) ) - }) - (ret, paginatedLogsResponse._2) - }) + )).toSeq + // Retrieve the scheduled executions for the given job + val scheduledExecutionsJson: Seq[JValue] = toFhirEngine.runningJobRegistry.getScheduledExecutions(jobId) + .map(id => JObject( + List( + "id" -> JString(id), + "scheduled" -> JBool(true) + ) + )).toSeq - case None => throw ResourceNotFound("Mapping job does not exists.", s"A mapping job with id $jobId does not exists") - } - } + Future.successful(runningExecutionsJson ++ scheduledExecutionsJson) - /** - * Returns the execution logs for a specific execution ID. - * - * @param projectId project id the job belongs to - * @param jobId job id - * @param executionId execution id - * @return the execution summary as a JSON object - */ - def getExecutionById(projectId: String, jobId: String, executionId: String): Future[JObject] = { - // Retrieve the job to validate its existence - jobRepository.getJob(projectId, jobId).flatMap { - case Some(_) => - // Get - logServiceClient.getExecutionById(projectId, jobId, executionId).map(executionJson=> { - logger.debug(s"Retrieved execution for projectId: $projectId, jobId: $jobId, executionId: $executionId") - // Add runningStatus field to the JSON object - executionJson ~ ("runningStatus" -> JBool(toFhirEngine.runningJobRegistry.getRunningExecutions(jobId).contains(executionId))) ~ ("scheduled" -> JBool(toFhirEngine.runningJobRegistry.isScheduled(jobId, executionId))) - }) - case None => - throw ResourceNotFound("Mapping job does not exist.", s"A mapping job with id $jobId does not exist") + case None => throw ResourceNotFound("Mapping job does not exists.", s"A mapping job with id $jobId does not exists") } } diff --git a/tofhir-server/src/main/scala/io/tofhir/server/util/LogServiceClient.scala b/tofhir-server/src/main/scala/io/tofhir/server/util/LogServiceClient.scala deleted file mode 100644 index f7eb9cb0..00000000 --- a/tofhir-server/src/main/scala/io/tofhir/server/util/LogServiceClient.scala +++ /dev/null @@ -1,110 +0,0 @@ -package io.tofhir.server.util - -import akka.http.scaladsl.Http -import akka.http.scaladsl.model.{HttpMethods, HttpRequest, Uri} -import io.tofhir.engine.Execution.actorSystem -import io.tofhir.engine.Execution.actorSystem.dispatcher -import io.tofhir.server.common.interceptor.ICORSHandler -import io.tofhir.common.model.Json4sSupport.formats -import org.json4s.JValue -import org.json4s.JsonAST.JObject -import org.json4s.jackson.JsonMethods - -import scala.concurrent.Future -import scala.concurrent.duration.DurationInt - -/** - * A client to connect to the log service - * // TODO handle exceptional cases in the responses - * - * @param logServiceEndpoint - */ -class LogServiceClient(logServiceEndpoint: String) { - val timeout = 20.seconds - - /** - * Retrieves logs for the executions associated to a specific job - * - * @param projectId - * @param jobId - * @param page desired number of page - * @param rowPerPage row count per page - * @param dateBefore last date of the filtered executions - * @param dateAfter start date of the filtered executions - * @param errorStatuses comma-separated list of error statuses to be used for filtering. - * @return A future of a tuple containing - * the details of individual executions as the first element - * total number of executions as the second element - * //TODO we can define a dedicated class representing the response type - */ - def getExecutions(projectId: String, jobId: String, page: String, rowPerPage: String, dateBefore: String, dateAfter: String, errorStatuses: String ): Future[(Seq[JValue], Long)] = { - val params = Map("page" -> page, - "dateBefore" -> dateBefore, - "dateAfter" -> dateAfter, - "errorStatuses" -> errorStatuses, - "rowPerPage" -> rowPerPage) - val uri: Uri = s"$logServiceEndpoint/projects/$projectId/jobs/$jobId/executions" - val request = HttpRequest( - method = HttpMethods.GET, - ).withUri(uri.withQuery(Uri.Query(params))) - - var countHeader: Long = 0 - Http().singleRequest(request) - .flatMap(resp => { - countHeader = resp.headers.find(_.name == ICORSHandler.X_TOTAL_COUNT_HEADER).map(_.value).get.toInt - resp.entity.toStrict(timeout) - }) - .map(strictEntity => { - val response = strictEntity.data.utf8String - (JsonMethods.parse(response).extract[Seq[JValue]], countHeader) - }) - } - - /** - * Retrieves logs for a specific execution - * - * @param projectId - * @param jobId - * @param executionId - * @return - */ - def getExecutionLogs(projectId: String, jobId: String, executionId: String): Future[Seq[JValue]] = { - val request = HttpRequest( - method = HttpMethods.GET, - uri = s"$logServiceEndpoint/projects/$projectId/jobs/$jobId/executions/$executionId/logs" - ) - - Http().singleRequest(request) - .flatMap(resp => { - resp.entity.toStrict(timeout) - }) - .map(strictEntity => { - val response = strictEntity.data.utf8String - JsonMethods.parse(response).extract[Seq[JValue]] - }) - } - - /** - * Retrieves execution details including mapping tasks, error status, start time, etc. - * - * @param projectId - * @param jobId - * @param executionId - * @return - */ - def getExecutionById(projectId: String, jobId: String, executionId: String): Future[JObject] = { - val request = HttpRequest( - method = HttpMethods.GET, - uri = s"$logServiceEndpoint/projects/$projectId/jobs/$jobId/executions/$executionId" - ) - - Http().singleRequest(request) - .flatMap { resp => resp.entity.toStrict(timeout) } - .map(strictEntity => { - println() - val response = strictEntity.data.utf8String - JsonMethods.parse(response).extract[JObject] - }) - - } -} diff --git a/tofhir-server/src/test/scala/io/tofhir/server/BaseEndpointTest.scala b/tofhir-server/src/test/scala/io/tofhir/server/BaseEndpointTest.scala index a35a4976..6926cc12 100644 --- a/tofhir-server/src/test/scala/io/tofhir/server/BaseEndpointTest.scala +++ b/tofhir-server/src/test/scala/io/tofhir/server/BaseEndpointTest.scala @@ -8,7 +8,7 @@ import io.tofhir.common.model.Json4sSupport.formats import io.tofhir.engine.config.ToFhirEngineConfig import io.tofhir.engine.util.FhirMappingJobFormatter.EnvironmentVariable import io.tofhir.engine.util.FileUtils -import io.tofhir.server.config.{LogServiceConfig, RedCapServiceConfig} +import io.tofhir.server.config.RedCapServiceConfig import io.tofhir.server.common.config.WebServerConfig import io.tofhir.server.endpoint.ToFhirServerEndpoint import io.tofhir.server.fhir.FhirDefinitionsConfig @@ -32,7 +32,6 @@ trait BaseEndpointTest extends AnyWordSpec with Matchers with ScalatestRouteTest val toFhirEngineConfig: ToFhirEngineConfig = new ToFhirEngineConfig(system.settings.config.getConfig("tofhir")) val webServerConfig = new WebServerConfig(system.settings.config.getConfig("webserver")) val fhirDefinitionsConfig = new FhirDefinitionsConfig(system.settings.config.getConfig("fhir")) - val logServiceConfig = new LogServiceConfig(system.settings.config.getConfig("log-service")) val redCapServiceConfig = new RedCapServiceConfig(system.settings.config.getConfig("tofhir-redcap")) // route endpoint var route: Route = _ @@ -82,7 +81,7 @@ trait BaseEndpointTest extends AnyWordSpec with Matchers with ScalatestRouteTest FileUtils.getPath(fhirDefinitionsConfig.codesystemsPath.get).toFile.mkdirs() FileUtils.getPath(fhirDefinitionsConfig.valuesetsPath.get).toFile.mkdirs() // initialize endpoint and route - val endpoint = new ToFhirServerEndpoint(toFhirEngineConfig, webServerConfig, fhirDefinitionsConfig, logServiceConfig, redCapServiceConfig) + val endpoint = new ToFhirServerEndpoint(toFhirEngineConfig, webServerConfig, fhirDefinitionsConfig, redCapServiceConfig) route = endpoint.toFHIRRoute } diff --git a/tofhir-server/src/test/scala/io/tofhir/server/service/ExecutionServiceTest.scala b/tofhir-server/src/test/scala/io/tofhir/server/service/ExecutionServiceTest.scala index 681bf270..effcec7d 100644 --- a/tofhir-server/src/test/scala/io/tofhir/server/service/ExecutionServiceTest.scala +++ b/tofhir-server/src/test/scala/io/tofhir/server/service/ExecutionServiceTest.scala @@ -45,7 +45,7 @@ class ExecutionServiceTest extends AsyncWordSpec with Matchers with BeforeAndAft val schemaRepository: SchemaFolderRepository = getMockSchemaRepository val mappingJobRepository: JobFolderRepository = getMockMappingJobRepository // the execution service instance for the test - val executionService: ExecutionService = new ExecutionService(mappingJobRepository, mappingRepository, schemaRepository, "") + val executionService: ExecutionService = new ExecutionService(mappingJobRepository, mappingRepository, schemaRepository) "The Execution Service" should { "should clear checkpoint directory" in {