[O11y] Migrate infraobs packages to ecs@mappings

packages/cassandra/_dev/build/build.yml

@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@v8.5.1
+    reference: "git@v8.11.0"

packages/cassandra/_dev/build/docs/README.md

@@ -79,6 +79,10 @@ Cassandra system logs from cassandra.log files.
 
 {{event "log"}}
 
+**ECS Field Reference**
+
+Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
+
 {{fields "log"}}
 
 ## Metrics
@@ -87,4 +91,8 @@ Cassandra metrics using jolokia agent installed on cassandra.
 
 {{event "metrics"}}
 
+**ECS Field Reference**
+
+Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
+
 {{fields "metrics"}}

packages/cassandra/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.14.0"
+  changes:
+    - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10171
 - version: "1.13.0"
   changes:
     - description: Add global filter on data_stream.dataset to improve performance.

packages/cassandra/data_stream/log/_dev/test/pipeline/test-cassandra.log-expected.json

@@ -3,17 +3,19 @@
         {
             "@timestamp": "2021-07-21T12:18:15.910Z",
             "ecs": {
-                "version": "8.5.1"
+                "version": "8.11.0"
             },
             "event": {
                 "category": [
                     "database"
                 ],
-                "ingested": "2023-10-11T21:02:06.332290589Z",
+                "ingested": "2024-06-17T12:09:55.250489457Z",
                 "kind": "event",
                 "module": "cassandra",
                 "original": "INFO  [main] 2021-07-21 12:18:15,910 YamlConfigurationLoader.java:92 - Configuration location: file:/C:/Users/kush.rana/Desktop/Projects/elasticconnectors/apache-cassandra-3.11.10/conf/cassandra.yaml",
-                "type": "info"
+                "type": [
+                    "info"
+                ]
             },
             "log": {
                 "level": "INFO",
@@ -42,17 +44,19 @@
                 }
             },
             "ecs": {
-                "version": "8.5.1"
+                "version": "8.11.0"
             },
             "event": {
                 "category": [
                     "database"
                 ],
-                "ingested": "2023-10-11T21:02:06.332297964Z",
+                "ingested": "2024-06-17T12:09:55.250550934Z",
                 "kind": "event",
                 "module": "cassandra",
                 "original": "INFO  [nioEventLoopGroup-2-1] 2021-07-21 12:23:32,856 Message.java:826 - Unexpected exception during request; channel = [id: 0xa6112238, L:/127.0.0.1:9042 - R:/127.0.0.1:60106]\njava.io.IOException: An existing connection was forcibly closed by the remote host\n\tat sun.nio.ch.SocketDispatcher.read0(Native Method) ~[na:1.8.0_291]\n\tat sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:43) ~[na:1.8.0_291]\n\tat sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[na:1.8.0_291]\n\tat sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[na:1.8.0_291]\n\tat sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:378) ~[na:1.8.0_291]\n\tat io.netty.buffer.PooledUnsafeDirectByteBuf.setBytes(PooledUnsafeDirectByteBuf.java:221) ~[netty-all-4.0.44.Final.jar:4.0.44.Final]\n\tat io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:899) ~[netty-all-4.0.44.Final.jar:4.0.44.Final]\n\tat io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:276) ~[netty-all-4.0.44.Final.jar:4.0.44.Final]\n\tat io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:119) ~[netty-all-4.0.44.Final.jar:4.0.44.Final]\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:643) [netty-all-4.0.44.Final.jar:4.0.44.Final]\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:566) [netty-all-4.0.44.Final.jar:4.0.44.Final]\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480) [netty-all-4.0.44.Final.jar:4.0.44.Final]\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442) [netty-all-4.0.44.Final.jar:4.0.44.Final]\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131) [netty-all-4.0.44.Final.jar:4.0.44.Final]\n\tat io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) [netty-all-4.0.44.Final.jar:4.0.44.Final]\n\tat java.lang.Thread.run(Thread.java:748) [na:1.8.0_291]",
-                "type": "info"
+                "type": [
+                    "info"
+                ]
             },
             "log": {
                 "level": "INFO",
@@ -81,17 +85,19 @@
                 }
             },
             "ecs": {
-                "version": "8.5.1"
+                "version": "8.11.0"
             },
             "event": {
                 "category": [
                     "database"
                 ],
-                "ingested": "2023-10-11T21:02:06.332299172Z",
+                "ingested": "2024-06-17T12:09:55.250555078Z",
                 "kind": "event",
                 "module": "cassandra",
                 "original": "INFO  [main] 2021-07-22 15:16:10,134 StorageService.java:681 - Token metadata: Normal Tokens:\nlocalhost/127.0.0.1:[-9213409579976581007, -9208536521948527928, -9138148778360337250, -9127679418115980016, -8976989747173636951, -8928929930114662189, -8850922876008716693, -8742572573666294013, -8682790054622712291, -8602448569428641857, -8427444260690062087, -8322248594442534434, -8320001287302042084, -8254990830747485697, -8249957862499495790, -8189195602746080394, -8129887952616066480, -8056374640546524116, -8038761382995613210, -8037170110133462140, -7992011367541245422, -7880071995925335357, -7749009042440224717, -7617052974926155684, -7579992376331554614, -7547752740136860669, -7483309149196458963, -7403928085586378365, -7295126747625344073, -7285427200042709300, -7208686794243768519, -7117693125722831897, -6883722024573005241, -6720612224987504455, -6717012773506031630, -6634852508780100068, -6552838126854940220, -6342653427563558228, -6320996911175515573, -5938863381584705682, -5937138028307335156, -5863008324229999355, -5557902421394732795, -5507288642909724159, -5359164966940526093, -5270856020393654677, -5140336908990033565, -5086699786885043460, -5074000871253631366, -5073978841190866944, -5033423903877820246, -4943085258474057592, -4910543332786614489, -4879770268169959108, -4850099303163390815, -4807455567471793547, -4719159438210854274, -4619781027025280965, -4544597174348575652, -4485428936415138994, -4472739122485282685, -4373079504728124492, -4181301628202154089, -4126388626984258138, -4023053100851092991, -3922047779752166298, -3906718855618646650, -3868634824115722238, -3745879358169402556, -3690097830756775997, -3670927950016768037, -3616747246231509319, -3591426209432189634, -3490702446505255801, -3483385134981416558, -3429491430707856745, -3416053347785698066, -3413943292199611161, -3382417634862376426, -3349947830775906953, -3296542850568272055, -3176473778862080919, -3166346545442410180, -3132772882761113905, -3043642279006642678, -2954123492179139102, -2863171390983702241, -2654073953489285272, -2590940177673686644, -2587080257265847771, -2251605852203586301, -2198878564462361531, -2145896005604227928, -2124367674677225113, -2100796974152514459, -2100767007153930839, -2048246436708511951, -1940231583736341613, -1918276191425512270, -1901156852629539284, -1880561834882743663, -1746302695032098428, -1656872679451617615, -1548673391155746482, -1488479865253002047, -1379516031698548835, -1375032214470929604, -1314590380943603420, -1308375119031032813, -1285371162065374453, -1236769884793871491, -1079440989079766338, -869710792032024432, -789236985842188234, -455105751942515560, -444051516445821516, -361900680170942718, -168328735467397181, -134714950793282931, 60826037083339962, 84541622509618720, 144554169302448103, 196774206633592259, 198562324344386301, 238610642455775463, 278056991129882380, 279880939653579207, 282592745265744974, 410971596053594328, 495924051707529203, 526093410792433875, 575034598008591456, 579720708854555015, 589085295330075427, 590435486579344463, 776031051171600786, 781641065494177208, 846457757495601745, 850257452008846361, 900686269816010653, 1051126519181924424, 1203321605454295227, 1203895566636331975, 1211465458560787736, 1225960882717793263, 1241774140877981683, 1292414550010972366, 1334361932455738343, 1337377210995562847, 1419081732672037948, 1419381122950575881, 1490964106631451820, 1639326652578045878, 1705770160551872440, 1832285348848161986, 1847497393690487631, 1879480685217394570, 1891538878205257739, 1989643415414534081, 1994910556424959679, 1998064184769937808, 2155104014043103344, 2220336291482261989, 2283828190095512675, 2306824176141659302, 2451278539468942429, 2598912773538613706, 2762278292210005010, 2773437737237250294, 2832174370654635432, 2934442852597210012, 2945700216164622238, 2952556039929943093, 3107437902315470865, 3123397929954224804, 3186025726071011271, 3188208477416257432, 3200767480243736192, 3215706690514302654, 3276105622621268635, 3365884355288559483, 3401162000846683197, 3490150748579176060, 3504387869819626580, 3724159158070359927, 3885983949513817526, 3928206314063268933, 4115069486609707213, 4242350427466708713, 4386862705409556464, 4404914535424634841, 4486289098334426088, 4615905347775520925, 4650730553761950776, 4664032552573343869, 4741710988150565521, 4824314289830954773, 4855150021956252527, 4979447180744678768, 5010835932690867774, 5135899429065919974, 5230763194691689473, 5318582515931199681, 5558115297723062617, 5588210318754588806, 5615741481489697481, 5681975707573416981, 5826288747232227118, 6135206485758434356, 6163357531758535338, 6192919223175738327, 6193503193865707083, 6251171521552312359, 6370634172051452277, 6440400015024985128, 6444714481840758288, 6619609681089611411, 6698206135058175326, 6723548174665139367, 6744705100400054244, 6777227783060561616, 6840248229536654533, 6903596833014731591, 6939254918732562615, 6986844194665101330, 6999783490815861150, 7081735174733692632, 7088499763485696293, 7164209991011322273, 7180706869985461496, 7369129159637618981, 7375016321335570664, 7518640636490654215, 7584590997557423209, 7665483325181367542, 7751011211020015083, 7808711095476824306, 7905270090323370693, 7930480210183871741, 8027805212938904497, 8086730665555500916, 8156173380772627797, 8455783018781361766, 8522975246241517657, 8525946248784902240, 8547978838832864285, 8580320166686867118, 8587722170086323264, 8726678858413102621, 8732783454936618116, 8734728266269135300, 8789489738850249699, 8799590254811299859, 8825243461020618263, 9047662432134874749, 9087960439561935209]\n",
-                "type": "info"
+                "type": [
+                    "info"
+                ]
             },
             "log": {
                 "level": "INFO",

packages/cassandra/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -8,7 +8,7 @@ processors:
       ignore_failure: true
   - set:
       field: ecs.version
-      value: 8.5.1
+      value: 8.11.0
       ignore_empty_value: true
       ignore_failure: true
   - set:
@@ -51,9 +51,9 @@ processors:
       source: >-
         def err_levels = ["FATAL", "ERROR", "WARN"];
         if (err_levels.contains(ctx.log.level)) {
-          ctx.event.type = "error";
+          ctx.event.type = ["error"];
         } else {
-          ctx.event.type = "info";
+          ctx.event.type = ["info"];
         }
   - script:
       description: Drops null/empty values recursively.

packages/cassandra/data_stream/log/sample_event.json

@@ -1,37 +1,37 @@
 {
-    "@timestamp": "2022-08-01T07:33:01.952Z",
+    "@timestamp": "2024-06-18T06:33:32.952Z",
     "agent": {
-        "ephemeral_id": "d6102ad8-04fe-46fa-bf67-cc98e3665348",
-        "hostname": "docker-fleet-agent",
-        "id": "d1a9277c-e5a2-4ee3-a973-18f2b62e3ad8",
+        "ephemeral_id": "b1e9fa09-5c73-45d9-b26f-184761635dd9",
+        "id": "97400795-188c-4140-a1ee-0002078c785d",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "7.15.0"
+        "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "cassandra.log",
         "namespace": "ep",
         "type": "logs"
     },
     "ecs": {
-        "version": "8.5.1"
+        "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "d1a9277c-e5a2-4ee3-a973-18f2b62e3ad8",
+        "id": "97400795-188c-4140-a1ee-0002078c785d",
         "snapshot": false,
-        "version": "7.15.0"
+        "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "database"
         ],
         "dataset": "cassandra.log",
-        "ingested": "2022-08-01T07:33:17Z",
+        "ingested": "2024-06-18T06:34:02Z",
         "kind": "event",
         "module": "cassandra",
-        "original": "INFO  [main] 2022-08-01 07:33:01,952 YamlConfigurationLoader.java:92 - Configuration location: file:/etc/cassandra/cassandra.yaml",
-        "type": "info"
+        "type": [
+            "info"
+        ]
     },
     "input": {
         "type": "log"

packages/cassandra/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml

@@ -3,7 +3,7 @@ description: Pipeline for parsing Cassandra metrics.
 processors:
   - set:
       field: ecs.version
-      value: 8.5.1
+      value: 8.11.0
       ignore_empty_value: true
       ignore_failure: true
   - set:

packages/cassandra/data_stream/metrics/fields/ecs.yml

@@ -1,32 +1,12 @@
-- external: ecs
-  name: ecs.version
-- external: ecs
-  name: error.message
-- external: ecs
-  name: event.category
-- external: ecs
-  name: event.created
-- external: ecs
-  name: event.dataset
-- external: ecs
-  name: event.kind
-- external: ecs
-  name: event.module
-- external: ecs
-  name: event.type
 - external: ecs
   name: service.address
   dimension: true
-- external: ecs
-  name: service.type
 - external: ecs
   name: host.name
   dimension: true
 - external: ecs
   name: agent.id
   dimension: true
-- external: ecs
-  name: cloud.project.id
 - external: ecs
   name: cloud.instance.id
   dimension: true