diff --git a/exporters/postgres/cht_sync_collector.yml b/exporters/postgres/cht_sync_collector.yml new file mode 100644 index 0000000..0b68bf8 --- /dev/null +++ b/exporters/postgres/cht_sync_collector.yml @@ -0,0 +1,75 @@ +collector_name: cht-sync + +# Update the schema and table names as needed +queries: + - query_name: couch2pg-query + query: | + SELECT + split_part(seq,'-',1) as sequence, + pending as pending, + CASE + WHEN updated_at < NOW() - INTERVAL '1 minute' THEN 0 + ELSE 1 + END AS liveness, + split_part(source,'/',2) as db, + split_part(source,'/',1) as cht_instance + FROM + v1.couchdb_progress + WHERE + source like '%/%' and + seq like '%-%' + ORDER BY + cht_instance, db + - query_name: dbt-latency + query: | + SELECT + EXTRACT(EPOCH FROM(couchdb.latest - dbt_root.latest)) AS dbt_latency + FROM + (SELECT MAX(saved_timestamp) as latest FROM v1.document_metadata) dbt_root, + (SELECT MAX(saved_timestamp) as latest FROM v1.couchdb) couchdb + - query_name: dbt-run-stats + query: | + SELECT + status, + execution_time, + name as table_name + FROM + v1.dbt_results + +metrics: + - metric_name: couch2pg_progress_sequence + type: counter + help: 'current sequence number for couch2pg' + key_labels: + - db + - cht_instance + values: [sequence] + query_ref: couch2pg-query + - metric_name: couch2pg_progress_pending + type: gauge + help: 'approximate number of changes left to sync from couch to postgres' + key_labels: + - db + - cht_instance + values: [pending] + query_ref: couch2pg-query + - metric_name: couch2pg_up + type: gauge + help: '1 if couch2pg is running and has updated in the last minute, 0 if not' + key_labels: + - db + - cht_instance + values: [liveness] + query_ref: couch2pg-query + - metric_name: dbt_latency + type: gauge + help: 'difference between last timestamp in dbt models and current time (seconds)' + values: [dbt_latency] + query_ref: dbt-latency + - metric_name: dbt_execution_time + type: gauge + help: 'dbt run last execution time (ms)' + key_labels: + - table_name + values: [execution_time] + query_ref: dbt-run-stats diff --git a/exporters/postgres/sql_servers_example.yml b/exporters/postgres/sql_servers_example.yml index 31e7d67..f5c2642 100644 --- a/exporters/postgres/sql_servers_example.yml +++ b/exporters/postgres/sql_servers_example.yml @@ -16,11 +16,11 @@ global: max_connection_lifetime: 10m collector_files: - - "/etc/sql_exporter/couch2pg_collector.yml" + - "/etc/sql_exporter/cht_sync_collector.yml" jobs: - job_name: db_targets - collectors: [couch2pg] + collectors: [cht-sync] # change this to [couch2pg] to monitor couch2pg enable_ping: true static_configs: - targets: @@ -28,5 +28,4 @@ jobs: # be sure each new server gets a unique name. A good rule of thumb is to use the name of the # sql server (eg "postgres-rds-prod", "postgres-rds-dev1" etc.) # 'postgres://USERNAME:PASSWORD@DB_SERVER_IP/DATABASE:PORT - "db1": 'postgres://cht_couch2pg:cht_couch2pg_password@172.17.0.1:5432/cht?sslmode=disable' # //NOSONAR - password is safe to commit - + "db1": 'postgres://postgres:password@172.17.0.1:5432/data?sslmode=disable' # //NOSONAR - password is safe to commit diff --git a/grafana/provisioning/dashboards/CHT/cht_admin_overview.json b/grafana/provisioning/dashboards/CHT/cht_admin_overview.json index b97d359..5aa7a36 100644 --- a/grafana/provisioning/dashboards/CHT/cht_admin_overview.json +++ b/grafana/provisioning/dashboards/CHT/cht_admin_overview.json @@ -1126,22 +1126,13 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "Number of CouchDB changes that have not yet been processed by Couch2pg", + "description": "Number of CouchDB changes that have not yet been processed by CHT Sync", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, - "displayName": "changes", - "links": [ - { - "title": "Couch2pg backlog", - "url": "/d/hkQUbyfVk/cht-admin-details?orgId=1&viewPanel=24&var-cht_instance=${cht_instance}&from=${__from}&to=${__to}" - } - ], "mappings": [], - "max": 100, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1149,14 +1140,6 @@ "color": "blue", "value": null }, - { - "color": "green", - "value": 0 - }, - { - "color": "#EAB839", - "value": 60 - }, { "color": "red", "value": 80 @@ -1173,15 +1156,10 @@ "x": 8, "y": 12 }, - "id": 49, - "links": [ - { - "targetBlank": true, - "title": "Couch2pg docs", - "url": "https://docs.communityhealthtoolkit.org/apps/tutorials/couch2pg-setup/" - } - ], + "id": 50, "options": { + "minVizHeight": 75, + "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ @@ -1202,47 +1180,17 @@ }, "editorMode": "builder", "exemplar": false, - "expr": "sum(cht_couchdb_update_sequence{instance=~\"$cht_instance\", db=~\"medic|sentinel|medic-users-meta\"})", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "sum(couch2pg_progress_sequence{cht_instance=~\"$cht_instance\", db=~\"medic|medic-sentinel|medic-users-meta\"})", - "hide": false, + "expr": "couch2pg_progress_pending{target=\"$cht_instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", "range": false, - "refId": "B" - }, - { - "datasource": { - "name": "Expression", - "type": "__expr__", - "uid": "__expr__" - }, - "expression": "$A - $B", - "hide": false, - "refId": "C", - "type": "math" - } - ], - "title": "Couch2pg Backlog", - "transformations": [ - { - "id": "filterByRefId", - "options": { - "include": "C" - } + "refId": "A", + "useBackend": false } ], + "title": "CHT Sync Backlog", "type": "gauge" }, { @@ -2303,8 +2251,7 @@ ], "refresh": "30s", "revision": 1, - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 39, "tags": [], "templating": { "list": [ @@ -2340,6 +2287,6 @@ "timezone": "", "title": "CHT Admin Overview", "uid": "oa2OfL-Vk", - "version": 37, + "version": 39, "weekStart": "" -} +} \ No newline at end of file diff --git a/index.js b/index.js new file mode 100644 index 0000000..04bf26e --- /dev/null +++ b/index.js @@ -0,0 +1 @@ +console.log('Happy developing ✨')