Leonardo Cantarella Kart Solution

.gitignore

@@ -0,0 +1 @@
+.vscode/

README.md

@@ -12,7 +12,47 @@ Mario would like to dispatch automatically the cars according to the level of da
 - scrap: cars that need to be demolished
 
 ## Challenge
-Help Mario implementing a system based of files put in a given directory 
+Help Mario implementing a system based of files sent via httpd
 (choose the format, remember to be able to distinguish the status in the last line)
 send to Kafka and using Kafka Stream perform a triage and sent to the appropriate topic
 Use kafka consumer to show the status of the queues
+
+# Prerequisites
+## Build Stream App
+To start the project run
+```bash
+cd kart-stream
+docker build . --tag kart
+```
+
+# Execution
+## Run
+```bash
+docker-compose -f kart.yml up
+```
+To send message to KART using curl
+
+```bash
+## Log message of car unrecoverable
+curl -d"Luigi SYS64738" http://localhost:9090
+
+## Log message of car for garage
+curl -d"Mario OK" http://localhost:9090
+```
+
+To stop press CTRL+C
+
+## Remove containers
+```bash
+docker-compose -f kart.yml down
+```
+# Implementation Details
+## Archetype
+```bash
+mvn archetype:generate \
+    -DarchetypeGroupId=org.apache.kafka \
+    -DarchetypeArtifactId=streams-quickstart-java \
+    -DarchetypeVersion=3.7.0 \
+    -DgroupId=tap.kart \
+    -DartifactId=kart-stream
+```

kart-logstash.conf

@@ -0,0 +1,20 @@
+input {
+  http {
+    id => "tap_http_in"
+    port => 9090
+  }
+}
+
+filter {
+    if [http][method] != "POST"{
+        drop {}
+    }
+}
+
+output {
+  kafka {
+    codec => json
+    topic_id => "kart-triage"
+    bootstrap_servers => "broker:9092"
+  }
+}

kart-stream/Dockerfile

@@ -0,0 +1,19 @@
+# Creating first container to compile Java code and create jar file
+FROM maven:3-openjdk-8 AS build
+WORKDIR /app
+
+# Copying pom.xml file to container
+COPY pom.xml .
+
+# Downloading dependencies
+RUN mvn -f ./pom.xml clean package
+
+COPY src src
+
+RUN mvn -f ./pom.xml package
+
+FROM apache/kafka:latest
+ENV PATH /opt/kafka/bin:$PATH
+WORKDIR /opt/kafka/
+COPY --from=build /app/target/*.jar  /opt/kafka/libs
+ENTRYPOINT [ "kafka-run-class.sh"]  

kart-stream/pom.xml

@@ -0,0 +1,146 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>streams.examples</groupId>
+    <artifactId>kart-stream</artifactId>
+    <version>1.0</version>
+    <packaging>jar</packaging>
+
+    <name>Kafka Streams Quickstart :: Java</name>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <kafka.version>3.7.0</kafka.version>
+        <slf4j.version>1.7.36</slf4j.version>
+    </properties>
+
+    <repositories>
+        <repository>
+            <id>apache.snapshots</id>
+            <name>Apache Development Snapshot Repository</name>
+            <url>https://repository.apache.org/content/repositories/snapshots/</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
+    <!--
+		Execute "mvn clean package -Pbuild-jar"
+		to build a jar file out of this project!
+	-->
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.1</version>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
+        </plugins>
+
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <artifactId>maven-compiler-plugin</artifactId>
+                    <configuration>
+                        <source>1.8</source>
+                        <target>1.8</target>
+                        <compilerId>jdt</compilerId>
+                    </configuration>
+                    <dependencies>
+                        <dependency>
+                            <groupId>org.eclipse.tycho</groupId>
+                            <artifactId>tycho-compiler-jdt</artifactId>
+                            <version>0.21.0</version>
+                        </dependency>
+                    </dependencies>
+                </plugin>
+                <plugin>
+                    <groupId>org.eclipse.m2e</groupId>
+                    <artifactId>lifecycle-mapping</artifactId>
+                    <version>1.0.0</version>
+                    <configuration>
+                        <lifecycleMappingMetadata>
+                            <pluginExecutions>
+                                <pluginExecution>
+                                    <pluginExecutionFilter>
+                                        <groupId>org.apache.maven.plugins</groupId>
+                                        <artifactId>maven-assembly-plugin</artifactId>
+                                        <versionRange>[2.4,)</versionRange>
+                                        <goals>
+                                            <goal>single</goal>
+                                        </goals>
+                                    </pluginExecutionFilter>
+                                    <action>
+                                        <ignore/>
+                                    </action>
+                                </pluginExecution>
+                                <pluginExecution>
+                                    <pluginExecutionFilter>
+                                        <groupId>org.apache.maven.plugins</groupId>
+                                        <artifactId>maven-compiler-plugin</artifactId>
+                                        <versionRange>[3.1,)</versionRange>
+                                        <goals>
+                                            <goal>testCompile</goal>
+                                            <goal>compile</goal>
+                                        </goals>
+                                    </pluginExecutionFilter>
+                                    <action>
+                                        <ignore/>
+                                    </action>
+                                </pluginExecution>
+                            </pluginExecutions>
+                        </lifecycleMappingMetadata>
+                    </configuration>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
+
+    <dependencies>
+        <!-- To enable console logging -->
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-reload4j</artifactId>
+            <version>${slf4j.version}</version>
+        </dependency>
+        <!-- Apache Kafka dependencies -->
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-streams</artifactId>
+            <version>${kafka.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.16.0</version>
+        </dependency>
+    </dependencies>
+</project>

kart-stream/src/main/java/tap/kart/kartStream.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package tap.kart;
+
+import org.apache.kafka.common.serialization.Serdes;
+import org.apache.kafka.streams.KafkaStreams;
+import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.Topology;
+import org.apache.kafka.streams.processor.TopicNameExtractor;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.ObjectCodec;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+
+
+import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
+
+
+public class kartStream {
+
+    public static void main(String[] args) {
+        Properties props = new Properties();
+        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "kartStream");
+        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "broker:9092");
+        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
+
+        final StreamsBuilder builder = new StreamsBuilder();
+        String inputStreamName="kart-triage";
+
+        /* 
+         * {"@timestamp":"2024-04-23T16:17:34.899108395Z","user_agent":{"original":"curl/7.81.0"},"url":{"path":"/","domain":"localhost","port":9090},"message":"test","http":{"method":"POST","version":"HTTP/1.1","request":{"mime_type":"application/x-www-form-urlencoded","body":{"bytes":"4"}}},"event":{"original":"test"},"@version":"1","host":{"ip":"172.25.0.1"}}
+        */
+
+        TopicNameExtractor<String, String> triageTopicExtractor = (key, value, recordContext) -> {
+            String dispatchTopic="kart-garage";
+            try {
+                ObjectMapper objectMapper = new ObjectMapper();
+                JsonNode jsonNode = objectMapper.readTree(value);
+                String message = jsonNode.get("message").asText();
+                if (message.contains("SYS64738")) 
+                    dispatchTopic="kart-scrapyard";
+                //System.out.println(value);
+                //System.out.println("Message:"+message);
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+
+            return(dispatchTopic);
+        };
+        builder.<String, String>stream(inputStreamName).to(triageTopicExtractor);
+
+        final Topology topology = builder.build();
+        final KafkaStreams streams = new KafkaStreams(topology, props);
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        // attach shutdown handler to catch control-c
+        Runtime.getRuntime().addShutdownHook(new Thread("streams-shutdown-hook") {
+            @Override
+            public void run() {
+                streams.close();
+                latch.countDown();
+            }
+        });
+
+        try {
+            streams.start();
+            latch.await();
+        } catch (Throwable e) {
+            System.exit(1);
+        }
+        System.exit(0);
+    }
+}

kart-stream/src/main/java/tap/kart/readme.md

@@ -0,0 +1,2 @@
+docker build . --tag kart
+tap.kart.kartStream

kart-stream/src/main/resources/log4j.properties

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=INFO, console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n

kart-stream/target/classes/log4j.properties

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=INFO, console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n