Updated sync data model. Added source to observation data model.

dabamos · Feb 26, 2024 · de0143c · de0143c
1 parent 42aa854
commit de0143c
Show file tree

Hide file tree

Showing 27 changed files with 335 additions and 271 deletions.
diff --git a/app/dmfs.f90 b/app/dmfs.f90
@@ -214,11 +214,12 @@ integer function read_config(app) result(rc)
         call dm_config_close(config)
     end function read_config
 
-    integer function read_observ(observ, node_id, sensor_id, debug) result(rc)
+    integer function read_observ(observ, node_id, sensor_id, source, debug) result(rc)
         !! Reads observation from file.
         type(observ_type), target, intent(inout)        :: observ    !! Observation to read.
         character(len=*),          intent(in)           :: node_id   !! Node id of observation.
         character(len=*),          intent(in)           :: sensor_id !! Sensor id of observation.
+        character(len=*),          intent(in)           :: source    !! Source of observation.
         logical,                   intent(in), optional :: debug     !! Output debug messages.
 
         integer                      :: delay
@@ -235,6 +236,7 @@ integer function read_observ(observ, node_id, sensor_id, debug) result(rc)
         observ%id        = dm_uuid4()
         observ%node_id   = node_id
         observ%sensor_id = sensor_id
+        observ%source    = source
         observ%timestamp = dm_time_now()
 
         if (observ%nrequests == 0) then
@@ -389,7 +391,7 @@ subroutine run(app)
                 end if
 
                 ! Read observation from file system.
-                rc = read_observ(observ, app%node, app%sensor, debug=debug)
+                rc = read_observ(observ, app%node, app%sensor, app%name, debug=debug)
 
                 if (debug) then
                     call dm_log(LOG_DEBUG, 'finished observ ' // trim(observ%name) // &

diff --git a/app/dmpipe.f90 b/app/dmpipe.f90
@@ -281,6 +281,7 @@ subroutine run(app)
                 observ%id        = dm_uuid4()
                 observ%node_id   = app%node
                 observ%sensor_id = app%sensor
+                observ%source    = app%name
                 observ%timestamp = dm_time_now()
 
                 if (observ%nrequests == 0) then

diff --git a/app/dmserial.f90 b/app/dmserial.f90
@@ -495,6 +495,7 @@ integer function run(app, tty) result(rc)
                 ! Initialise observation.
                 observ%node_id   = app%node
                 observ%sensor_id = app%sensor
+                observ%source    = app%name
                 observ%path      = trim(app%tty)
 
                 if (debug) then

diff --git a/app/dmsync.f90 b/app/dmsync.f90
@@ -482,9 +482,9 @@ integer function run(app, db, sem) result(rc)
                     end select code_select
 
                     ! Update sync data.
-                    syncs(i)%timestamp = dm_time_now()          ! Time of sync attempt.
-                    syncs(i)%code      = responses(i)%code      ! Server status code.
-                    syncs(i)%nattempts = syncs(i)%nattempts + 1 ! Number of sync attempts.
+                    syncs(i)%timestamp = dm_time_now()         ! Time of sync attempt.
+                    syncs(i)%code      = responses(i)%code     ! Server status code.
+                    syncs(i)%attempts  = syncs(i)%attempts + 1 ! Number of sync attempts.
 
                     ! Insert or replace the sync data in database. If the database
                     ! is busy, try up to `APP_DB_NATTEMPTS` times, then abort.

diff --git a/config/dmfs.conf.sample b/config/dmfs.conf.sample
@@ -33,7 +33,7 @@ observs = {
     {
         name = "dummy-observ",
         target_id = "dummy-target",
-        receivers = { "dmfs", "dmdb" },
+        receivers = { "dmdb" },
         requests = {
             {
                 request = path,

diff --git a/config/dmpipe.conf.sample b/config/dmpipe.conf.sample
@@ -26,7 +26,7 @@
 observ = {
     name = "dummy-observ",
     target_id = "dummy-target",
-    receivers = { "dmpipe", "dmrecv" },
+    receivers = { "dmrecv" },
     requests = {
         --
         -- Read CPU temperature.

diff --git a/config/dmserial.conf.sample b/config/dmserial.conf.sample
@@ -71,7 +71,7 @@ observs = {
         --
         name = "meter",
         target_id = "meter",
-        receivers = { "dmserial", "dmdb" },
+        receivers = { "dmdb" },
         requests = {
             {
                 request = "Meter" .. del,

diff --git a/guide/guide.adoc b/guide/guide.adoc
@@ -3760,6 +3760,7 @@ NODE%Z=0.0,
 | `sensor_id` | string  | 32   | Sensor id (`-0-9A-Za-z`).
 | `target_id` | string  | 32   | Target id (`-0-9A-Za-z`).
 | `name`      | string  | 32   | Observation name (`-0-9A-Za-z`).
+| `source`    | string  | 32   | Observation source (`-0-9A-Za-z`).
 | `timestamp` | string  | 32   | Date and time of observation (ISO 8601).
 | `path`      | string  | 32   | Path of TTY/PTY device.
 | `priority`  | integer | 4    | Message queue priority (>= 0).
@@ -3827,73 +3828,74 @@ NODE%Z=0.0,
 | 3         | `sensor_id`  | Sensor id.
 | 4         | `target_id`  | Target id.
 | 5         | `name`       | Observation name.
-| 6         | `timestamp`  | Date and time of observation.
-| 7         | `path`       | Path of TTY/PTY device.
-| 8         | `priority`   | Message queue priority.
-| 9         | `error`      | Error code.
-| 10        | `next`       | Cursor of receiver list (0 to 16).
-| 11        | `nreceivers` | Number of receivers (0 to 16).
-| 12        | `nrequests`  | Number of sensor requests (0 to 8).
-| 13 – 28   | `receivers`  | Array of receiver names (16).
-| 13        | `receiver`   | Receiver 1.
-| 14        | `receiver`   | Receiver 2.
-| 15        | `receiver`   | Receiver 3.
-| 16        | `receiver`   | Receiver 4.
-| 17        | `receiver`   | Receiver 5.
-| 18        | `receiver`   | Receiver 6.
-| 19        | `receiver`   | Receiver 7.
-| 20        | `receiver`   | Receiver 8.
-| 21        | `receiver`   | Receiver 9.
-| 22        | `receiver`   | Receiver 10.
-| 23        | `receiver`   | Receiver 11.
-| 24        | `receiver`   | Receiver 12.
-| 25        | `receiver`   | Receiver 13.
-| 26        | `receiver`   | Receiver 14.
-| 27        | `receiver`   | Receiver 15.
-| 28        | `receiver`   | Receiver 16.
-| 29 – 764  | `requests`   | Array of requests (8).
-| 29 – 103  | `request`    | Request 1.
-| 29        | `timestamp`  | Date and time of request.
-| 30        | `request`    | Raw request to sensor.
-| 31        | `response`   | Raw response of sensor.
-| 32        | `delimiter`  | Request delimiter.
-| 33        | `pattern`    | Regular expression pattern that describes the raw response.
-| 34        | `delay`      | Delay in mseconds to wait after the request.
-| 35        | `error`      | Error code.
-| 36        | `mode`       | Request mode.
-| 37        | `retries`    | Number of retries performed.
-| 38        | `state`      | Request state.
-| 39        | `timeout`    | Request timeout in mseconds.
-| 40        | `nresponses` | Number of responses (0 to 16).
-| 41 – 120  | `responses`  | Array of responses (16).
-| 41 – 45   | `response`   | Response 1.
-| 41        | `name`       | Response name.
-| 42        | `unit`       | Response unit.
-| 43        | `type`       | Response value type.
-| 44        | `error`      | Response error.
-| 45        | `value`      | Response value.
-| 46 – 50   | `response`   | Response 2.
-| 51 – 55   | `response`   | Response 3.
-| 56 – 60   | `response`   | Response 4.
-| 61 – 65   | `response`   | Response 5.
-| 66 – 70   | `response`   | Response 6.
-| 71 – 75   | `response`   | Response 7.
-| 76 – 80   | `response`   | Response 8.
-| 81 – 85   | `response`   | Response 9.
-| 86 – 90   | `response`   | Response 10.
-| 91 – 95   | `response`   | Response 11.
-| 96 – 100  | `response`   | Response 12.
-| 101 – 105 | `response`   | Response 13.
-| 106 – 110 | `response`   | Response 14.
-| 111 – 115 | `response`   | Response 15.
-| 116 – 120 | `response`   | Response 16.
-| 121 – 212 | `request`    | Request 2.
-| 213 – 304 | `request`    | Request 3.
-| 305 – 396 | `request`    | Request 4.
-| 397 – 488 | `request`    | Request 5.
-| 489 – 580 | `request`    | Request 6.
-| 581 – 672 | `request`    | Request 7.
-| 673 – 764 | `request`    | Request 8.
+| 6         | `source`     | Observation source.
+| 7         | `timestamp`  | Date and time of observation.
+| 8         | `path`       | Path of TTY/PTY device.
+| 9         | `priority`   | Message queue priority.
+| 10        | `error`      | Error code.
+| 11        | `next`       | Cursor of receiver list (0 to 16).
+| 12        | `nreceivers` | Number of receivers (0 to 16).
+| 13        | `nrequests`  | Number of sensor requests (0 to 8).
+| 14 – 29   | `receivers`  | Array of receiver names (16).
+| 14        | `receiver`   | Receiver 1.
+| 15        | `receiver`   | Receiver 2.
+| 16        | `receiver`   | Receiver 3.
+| 17        | `receiver`   | Receiver 4.
+| 18        | `receiver`   | Receiver 5.
+| 19        | `receiver`   | Receiver 6.
+| 20        | `receiver`   | Receiver 7.
+| 21        | `receiver`   | Receiver 8.
+| 22        | `receiver`   | Receiver 9.
+| 23        | `receiver`   | Receiver 10.
+| 24        | `receiver`   | Receiver 11.
+| 25        | `receiver`   | Receiver 12.
+| 26        | `receiver`   | Receiver 13.
+| 27        | `receiver`   | Receiver 14.
+| 28        | `receiver`   | Receiver 15.
+| 29        | `receiver`   | Receiver 16.
+| 30 – 765  | `requests`   | Array of requests (8).
+| 30 – 104  | `request`    | Request 1.
+| 30        | `timestamp`  | Date and time of request.
+| 31        | `request`    | Raw request to sensor.
+| 32        | `response`   | Raw response of sensor.
+| 33        | `delimiter`  | Request delimiter.
+| 34        | `pattern`    | Regular expression pattern that describes the raw response.
+| 35        | `delay`      | Delay in mseconds to wait after the request.
+| 36        | `error`      | Error code.
+| 37        | `mode`       | Request mode.
+| 38        | `retries`    | Number of retries performed.
+| 39        | `state`      | Request state.
+| 40        | `timeout`    | Request timeout in mseconds.
+| 41        | `nresponses` | Number of responses (0 to 16).
+| 42 – 121  | `responses`  | Array of responses (16).
+| 42 – 46   | `response`   | Response 1.
+| 42        | `name`       | Response name.
+| 43        | `unit`       | Response unit.
+| 44        | `type`       | Response value type.
+| 45        | `error`      | Response error.
+| 46        | `value`      | Response value.
+| 47 – 51   | `response`   | Response 2.
+| 52 – 56   | `response`   | Response 3.
+| 57 – 61   | `response`   | Response 4.
+| 62 – 66   | `response`   | Response 5.
+| 67 – 71   | `response`   | Response 6.
+| 72 – 76   | `response`   | Response 7.
+| 77 – 81   | `response`   | Response 8.
+| 82 – 86   | `response`   | Response 9.
+| 87 – 91   | `response`   | Response 10.
+| 92 – 96   | `response`   | Response 11.
+| 97 – 101  | `response`   | Response 12.
+| 102 – 106 | `response`   | Response 13.
+| 107 – 111 | `response`   | Response 14.
+| 112 – 116 | `response`   | Response 15.
+| 117 – 121 | `response`   | Response 16.
+| 122 – 213 | `request`    | Request 2.
+| 214 – 305 | `request`    | Request 3.
+| 306 – 397 | `request`    | Request 4.
+| 398 – 489 | `request`    | Request 5.
+| 490 – 581 | `request`    | Request 6.
+| 582 – 673 | `request`    | Request 7.
+| 674 – 765 | `request`    | Request 8.
 |===
 
 .HDF5 [[data-observ-hdf5]]
@@ -3908,6 +3910,7 @@ The HDF5 data-set description is too large to be fully shown in this document.
   "sensor_id": "dummy-sensor",
   "target_id": "dummy-target",
   "name": "dummy-observ",
+  "source": "dmdummy",
   "timestamp": "1970-01-01T00:00:00.000000+00:00",
   "path": "/dev/null",
   "priority": 0,
@@ -3955,9 +3958,10 @@ The HDF5 data-set description is too large to be fully shown in this document.
   node_id = "dummy-node",
   sensor_id = "dummy-sensor",
   target_id = "dummy-target",
+  name = "dummy-observ",
+  source = "dmdummy",
   timestamp = "1970-01-01T00:00:00.000000+00:00",
   path = "/dev/null",
-  name = "dummy-observ",
   error = 0,
   next = 1,
   priority = 0,
@@ -4000,6 +4004,7 @@ OBSERV%NODE_ID="dummy-node",
 OBSERV%SENSOR_ID="dummy-sensor",
 OBSERV%TARGET_ID="dummy-target",
 OBSERV%NAME="dummy-observ",
+OBSERV%SOURCE="dmdummy",
 OBSERV%TIMESTAMP="1970-01-01T00:00:00.000000+00:00",
 OBSERV%PATH="/dev/null",
 OBSERV%PRIORITY=0,

diff --git a/guide/resources/images/observ.svg b/guide/resources/images/observ.svg
diff --git a/src/dm_csv.f90 b/src/dm_csv.f90
@@ -228,6 +228,7 @@ function dm_csv_header_observ(separator) result(header)
                  'sensor_id'  // s // &
                  'target_id'  // s // &
                  'name'       // s // &
+                 'source'     // s // &
                  'timestamp'  // s // &
                  'tty'        // s // &
                  'priority'   // s // &
@@ -553,6 +554,7 @@ function csv_from_observ(observ, separator) result(csv)
               trim(observ%sensor_id)     // s // &
               trim(observ%target_id)     // s // &
               trim(observ%name)          // s // &
+              trim(observ%source)        // s // &
               trim(observ%timestamp)     // s // &
               trim(observ%path)          // s // &
               dm_itoa(observ%priority)   // s // &
@@ -1084,6 +1086,7 @@ integer function csv_read_observ(observ, unit, separator, quote) result(rc)
         rc = csv_next(buffer, observ%sensor_id,  s, n, p, q); if (rc /= E_NONE) return
         rc = csv_next(buffer, observ%target_id,  s, n, p, q); if (rc /= E_NONE) return
         rc = csv_next(buffer, observ%name,       s, n, p, q); if (rc /= E_NONE) return
+        rc = csv_next(buffer, observ%source,     s, n, p, q); if (rc /= E_NONE) return
         rc = csv_next(buffer, observ%timestamp,  s, n, p, q); if (rc /= E_NONE) return
         rc = csv_next(buffer, observ%path,       s, n, p, q); if (rc /= E_NONE) return
         rc = csv_next(buffer, observ%priority,   s, n, p, q); if (rc /= E_NONE) return
@@ -1537,12 +1540,13 @@ integer function csv_write_observ(observ, unit, header, separator) result(rc)
             if (stat /= 0) return
         end if
 
-        write (unit_, '(14a, 4(i0, a), i0)', advance='no', iostat=stat) &
+        write (unit_, '(16a, 4(i0, a), i0)', advance='no', iostat=stat) &
               trim(observ%id),        s, &
               trim(observ%node_id),   s, &
               trim(observ%sensor_id), s, &
               trim(observ%target_id), s, &
               trim(observ%name),      s, &
+              trim(observ%source),    s, &
               trim(observ%timestamp), s, &
               trim(observ%path),      s, &
               observ%priority,        s, &

diff --git a/src/dm_db.f90 b/src/dm_db.f90
@@ -1605,13 +1605,14 @@ integer function dm_db_insert_observ(db, observ, db_stmt) result(rc)
             if (sqlite3_bind_text(stmt,  3, trim(observ%sensor_id)) /= SQLITE_OK) exit sql_block
             if (sqlite3_bind_text(stmt,  4, trim(observ%target_id)) /= SQLITE_OK) exit sql_block
             if (sqlite3_bind_text(stmt,  5, trim(observ%name))      /= SQLITE_OK) exit sql_block
-            if (sqlite3_bind_text(stmt,  6, trim(observ%timestamp)) /= SQLITE_OK) exit sql_block
-            if (sqlite3_bind_text(stmt,  7, trim(observ%path))      /= SQLITE_OK) exit sql_block
-            if (sqlite3_bind_int (stmt,  8, observ%priority)        /= SQLITE_OK) exit sql_block
-            if (sqlite3_bind_int (stmt,  9, observ%error)           /= SQLITE_OK) exit sql_block
-            if (sqlite3_bind_int (stmt, 10, observ%next)            /= SQLITE_OK) exit sql_block
-            if (sqlite3_bind_int (stmt, 11, observ%nreceivers)      /= SQLITE_OK) exit sql_block
-            if (sqlite3_bind_int (stmt, 12, observ%nrequests)       /= SQLITE_OK) exit sql_block
+            if (sqlite3_bind_text(stmt,  6, trim(observ%source))    /= SQLITE_OK) exit sql_block
+            if (sqlite3_bind_text(stmt,  7, trim(observ%timestamp)) /= SQLITE_OK) exit sql_block
+            if (sqlite3_bind_text(stmt,  8, trim(observ%path))      /= SQLITE_OK) exit sql_block
+            if (sqlite3_bind_int (stmt,  9, observ%priority)        /= SQLITE_OK) exit sql_block
+            if (sqlite3_bind_int (stmt, 10, observ%error)           /= SQLITE_OK) exit sql_block
+            if (sqlite3_bind_int (stmt, 11, observ%next)            /= SQLITE_OK) exit sql_block
+            if (sqlite3_bind_int (stmt, 12, observ%nreceivers)      /= SQLITE_OK) exit sql_block
+            if (sqlite3_bind_int (stmt, 13, observ%nrequests)       /= SQLITE_OK) exit sql_block
 
             rc = E_DB_STEP
             if (sqlite3_step(stmt) /= SQLITE_DONE) exit sql_block
@@ -4954,7 +4955,7 @@ integer function db_insert_sync(db, sync, query) result(rc)
             if (sqlite3_bind_text(stmt, 1, trim(sync%id))        /= SQLITE_OK) exit sql_block
             if (sqlite3_bind_text(stmt, 2, trim(sync%timestamp)) /= SQLITE_OK) exit sql_block
             if (sqlite3_bind_int (stmt, 3, sync%code)            /= SQLITE_OK) exit sql_block
-            if (sqlite3_bind_int (stmt, 4, sync%nattempts)       /= SQLITE_OK) exit sql_block
+            if (sqlite3_bind_int (stmt, 4, sync%attempts)        /= SQLITE_OK) exit sql_block
 
             rc = E_DB_STEP
             if (sqlite3_step(stmt) /= SQLITE_DONE) exit sql_block
@@ -5118,25 +5119,27 @@ integer function db_next_row_observ(stmt, observ, validate) result(rc)
             if (sqlite3_column_type(stmt,  4) /= SQLITE_TEXT)    return
             if (sqlite3_column_type(stmt,  5) /= SQLITE_TEXT)    return
             if (sqlite3_column_type(stmt,  6) /= SQLITE_TEXT)    return
-            if (sqlite3_column_type(stmt,  7) /= SQLITE_INTEGER) return
+            if (sqlite3_column_type(stmt,  7) /= SQLITE_TEXT)    return
             if (sqlite3_column_type(stmt,  8) /= SQLITE_INTEGER) return
             if (sqlite3_column_type(stmt,  9) /= SQLITE_INTEGER) return
             if (sqlite3_column_type(stmt, 10) /= SQLITE_INTEGER) return
             if (sqlite3_column_type(stmt, 11) /= SQLITE_INTEGER) return
+            if (sqlite3_column_type(stmt, 12) /= SQLITE_INTEGER) return
         end if
 
         observ%id         = sqlite3_column_text(stmt,  0)
         observ%node_id    = sqlite3_column_text(stmt,  1)
         observ%sensor_id  = sqlite3_column_text(stmt,  2)
         observ%target_id  = sqlite3_column_text(stmt,  3)
         observ%name       = sqlite3_column_text(stmt,  4)
-        observ%timestamp  = sqlite3_column_text(stmt,  5)
-        observ%path       = sqlite3_column_text(stmt,  6)
-        observ%priority   = sqlite3_column_int (stmt,  7)
-        observ%error      = sqlite3_column_int (stmt,  8)
-        observ%next       = sqlite3_column_int (stmt,  9)
-        observ%nreceivers = sqlite3_column_int (stmt, 10)
-        observ%nrequests  = sqlite3_column_int (stmt, 11)
+        observ%source     = sqlite3_column_text(stmt,  5)
+        observ%timestamp  = sqlite3_column_text(stmt,  6)
+        observ%path       = sqlite3_column_text(stmt,  7)
+        observ%priority   = sqlite3_column_int (stmt,  8)
+        observ%error      = sqlite3_column_int (stmt,  9)
+        observ%next       = sqlite3_column_int (stmt, 10)
+        observ%nreceivers = sqlite3_column_int (stmt, 11)
+        observ%nrequests  = sqlite3_column_int (stmt, 12)
 
         rc = E_NONE
     end function db_next_row_observ
@@ -5272,9 +5275,9 @@ integer function db_next_row_sync(stmt, sync) result(rc)
         end if
 
         if (sqlite3_column_type(stmt, 3) == SQLITE_INTEGER) then
-            sync%nattempts = sqlite3_column_int(stmt, 3)
+            sync%attempts = sqlite3_column_int(stmt, 3)
         else
-            sync%nattempts = 0
+            sync%attempts = 0
         end if
 
         rc = E_NONE