diff --git a/DEPLOY.md b/DEPLOY.md deleted file mode 100644 index cfc21f1..0000000 --- a/DEPLOY.md +++ /dev/null @@ -1,19 +0,0 @@ -Publish Grok Distribution Package ------- - -Publish Maven artifact ------------- - -**Publish to snapshot repository** - - mvn -DperformRelease=true deploy - - -**Publish to release repository** - - mvn -DperformRelease=true release:clean - mvn -DperformRelease=true release:prepare - mvn -DperformRelease=true release:perform - -Artifact is now in staging repository. -Connect https://oss.sonatype.org/ , select staging repository and click "close" -> "release" will finally release it. diff --git a/README.md b/README.md index c006878..12dcf11 100644 --- a/README.md +++ b/README.md @@ -31,33 +31,33 @@ compile "io.thekraken:grok:0.1.5" Example of how to use java-grok: ```java -/* Create a new grok instance */ -Grok grok = Grok.create(); +/* Create a new grokCompiler instance */ +GrokCompiler grokCompiler = GrokCompiler.newInstance(); +grokCompiler.registerDefaultPatterns(); /* Grok pattern to compile, here httpd logs */ -grok.compile("%{COMBINEDAPACHELOG}"); +final Grok grok = grokCompiler.compile("%{COMBINEDAPACHELOG}"); /* Line of log to match */ String log = "112.169.19.192 - - [06/Mar/2013:01:36:30 +0900] \"GET / HTTP/1.1\" 200 44346 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.152 Safari/537.22\""; Match gm = grok.match(log); -gm.captures(); - -/* Get the output */ -System.out.println(gm.toJson()); /* Get the map with matches */ -Map myMap = gm.toMap(); +final Map capture = gm.capture(); ``` ### Build Java Grok -Java Grok support Gradle: ``./gradlew build`` +Java Grok support Gradle: `./gradlew assemble` ### Getting help [Mailling List](https://groups.google.com/forum/#!forum/java-grok) -### Thankx to +### Thanks to + * [@joschi](https://github.com/joschi) + * [@keitaf](https://github.com/keitaf) + * [@anthonycorbacho](https://github.com/anthonycorbacho) * [@nokk](https://github.com/nokk) * [@wouterdb](https://github.com/wouterdb) * [@Leemoonsoo](https://github.com/Leemoonsoo) diff --git a/build.gradle b/build.gradle index 3ac9712..4fe7485 100644 --- a/build.gradle +++ b/build.gradle @@ -4,6 +4,8 @@ apply { plugin "maven-publish" plugin "checkstyle" plugin "jacoco" + plugin "idea" + plugin "eclipse" } task wrapper(type: Wrapper) { @@ -16,29 +18,29 @@ repositories { } sourceCompatibility = JavaVersion.VERSION_1_8 -group = "oi.thekraken" -archivesBaseName = "grok" -version = '0.1.6-SNAPSHOT' +group = "oi.krakens" +archivesBaseName = "java-grok" +version = '0.1.9-SNAPSHOT' ext { checkStyleToolVersion = "6.19" jacocoToolVersion = "0.7.9" - namedRegexpVersion = "0.2.4" commonsLang3Version = "3.7" - gsonVersion = "2.7" log4jVersion = "1.7.21" junitVersion = "4.12" + assertjVersion = "3.9.1" + guavaVersion = "24.0-jre" } dependencies { - compile "com.github.tony19:named-regexp:$namedRegexpVersion" compile "org.apache.commons:commons-lang3:$commonsLang3Version" - compile "com.google.code.gson:gson:$gsonVersion" compile "org.slf4j:slf4j-api:$log4jVersion" testCompile "junit:junit:$junitVersion" + testCompile "org.assertj:assertj-core:$assertjVersion" + testCompile "com.google.guava:guava:$guavaVersion" } checkstyle { diff --git a/extra/checkstyle/checkstyle.xml b/extra/checkstyle/checkstyle.xml index f8ba7f7..8736d4e 100644 --- a/extra/checkstyle/checkstyle.xml +++ b/extra/checkstyle/checkstyle.xml @@ -157,7 +157,6 @@ - diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..cc302f9 --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +version=0.1.9-SNAPSHOT \ No newline at end of file diff --git a/gradle/publishArtifact.gradle b/gradle/publishArtifact.gradle deleted file mode 100644 index aaa627d..0000000 --- a/gradle/publishArtifact.gradle +++ /dev/null @@ -1,85 +0,0 @@ -apply plugin: 'maven' -apply plugin: 'signing' - -task javadocJar(type: Jar) { - classifier = 'javadoc' - from javadoc -} - -task sourcesJar(type: Jar, dependsOn:classes) { - classifier = 'sources' - from sourceSets.main.allJava -} - -artifacts { - archives jar, javadocJar, sourcesJar -} - -signing { - sign configurations.archives -} - -/** Publish to Sonatype */ -def customizePom(pom, gradleProject) { - pom.whenConfigured { generatedPom -> - // add all items necessary for maven central publication - generatedPom.project { - name = 'Grok' - description = 'Simple API that allows you to easily parse logs and other files' - url = 'http://grok.nflabs.com' - packaging 'jar' - organization { - name = 'Krakken' - url = 'http://release.thekraken.io/' - } - licenses { - license { - name 'The Apache Software License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - distribution 'repo' - } - } - scm { - url = 'http://github.com/thekrakken/java-grok' - connection = 'scm:git:git@github.com:thekrakken/java-grok.git' - developerConnection = 'scm:git:git@github.com:thekrakken/java-grok.git' - } - developers { - developer { - id = 'anthonycorbacho' - name = 'Anthony Corbacho' - email = 'corbacho.anthony@gmail.com' - } - } - } - } -} - -repositories { - flatDir { - name "fileRepo" - dirs "repo" - } -} - -uploadArchives { - repositories { - //add project.repositories.fileRepo - mavenDeployer { - //beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - customizePom(pom, project) - repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { - authentication(userName: sonatypeUsername, password: sonatypePassword) - } - snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") { - authentication(userName: sonatypeUsername, password: sonatypePassword) - } - } - } -} - -install { - repositories.mavenInstaller { - customizePom(pom, project) - } -} \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ea720f9..b45acb9 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,9 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists +<<<<<<< HEAD distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-bin.zip +======= +>>>>>>> 84481ab507dcb89bf4ac15a00de47a2998c36cf3 diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 4e92e1d..0000000 --- a/pom.xml +++ /dev/null @@ -1,276 +0,0 @@ - - 4.0.0 - - io.thekraken - grok - 0.1.6-SNAPSHOT - jar - Grok - Simple API that allows you to easily parse logs and other files - http://maven.apache.org - - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - org.sonatype.oss - oss-parent - 7 - - - - - Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0 - repo - - - - - - anthony-corbacho - Anthony Corbacho - corbacho.anthony@gmail.com - Thekraken - http://release.thekraken.io - https://github.com/anthonycorbacho - - - - - scm:git:git@github.com:thekrakken/java-grok.git - scm:git:git@github.com:thekrakken/java-grok.git - scm:git:git@github.com:thekrakken/java-grok.git - - - - Travis - https://travis-ci.org/thekrakken/java-grok - - - - Github - https://github.com/thekrakken/java-grok/issues - - - - 4.11 - 3.1 - 2.2.2 - - 1.7 - - - - - junit - junit - ${junit.version} - test - - - org.apache.commons - commons-lang3 - ${common.version} - compile - - - - com.google.code.gson - gson - ${gson.version} - compile - - - - org.slf4j - slf4j-api - 1.7.5 - compile - - - - - - - maven-compiler-plugin - 3.1 - - ${java.version} - ${java.version} - - - - - maven-javadoc-plugin - 2.9.1 - - - - - aggregate - - aggregate - - site - - - - - - - - maven-scm-plugin - 1.8.1 - - developerConnection - branch-0.1 - branch - - - - - maven-enforcer-plugin - 1.3.1 - - - enforce - - - - - true - - - enforce - - - - - - - - - - maven-dependency-plugin - 2.8 - - - maven-surefire-plugin - 2.16 - - -Xmx2g -Xms1g - - - - maven-assembly-plugin - 2.4 - - - org.codehaus.mojo - exec-maven-plugin - 1.2.1 - - - - org.apache.maven.plugins - maven-clean-plugin - 2.4.1 - - - - drivers - false - - - - - - - - - - - build-distr - - false - - - - - - maven-surefire-plugin - - true - - - - maven-assembly-plugin - - - make-assembly - package - - single - - - - - - - - - - - publish-distr - - false - - - - - maven-surefire-plugin - - true - - - - - - - - release-sign-artifacts - - - performRelease - true - - - - - - maven-gpg-plugin - 1.4 - - - sign-artifacts - verify - - sign - - - - - - - - - diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..2adeea5 --- /dev/null +++ b/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'java-grok' diff --git a/src/main/java/io/krakens/grok/api/Converter.java b/src/main/java/io/krakens/grok/api/Converter.java index 5f7c4d9..5480c62 100644 --- a/src/main/java/io/krakens/grok/api/Converter.java +++ b/src/main/java/io/krakens/grok/api/Converter.java @@ -1,11 +1,22 @@ package io.krakens.grok.api; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.Locale; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalAccessor; +import java.util.AbstractMap; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Convert String argument to the right type. @@ -13,189 +24,132 @@ */ public class Converter { - public static Map> converters = new HashMap>(); - public static Locale locale = Locale.ENGLISH; - - static { - converters.put("byte", new ByteConverter()); - converters.put("boolean", new BooleanConverter()); - converters.put("short", new ShortConverter()); - converters.put("int", new IntegerConverter()); - converters.put("long", new LongConverter()); - converters.put("float", new FloatConverter()); - converters.put("double", new DoubleConverter()); - converters.put("date", new DateConverter()); - converters.put("datetime", new DateConverter()); - converters.put("string", new StringConverter()); + public enum Type { + BYTE(Byte::valueOf), + BOOLEAN(Boolean::valueOf), + SHORT(Short::valueOf), + INT(Integer::valueOf, "integer"), + LONG(Long::valueOf), + FLOAT(Float::valueOf), + DOUBLE(Double::valueOf), + DATETIME(new DateConverter(), "date"), + STRING(v -> v, "text"); - } - - private static IConverter getConverter(String key) throws Exception { - IConverter converter = converters.get(key); - if (converter == null) { - throw new Exception("Invalid data type :" + key); - } - return converter; - } + public final IConverter converter; + public final List aliases; - public static KeyValue convert(String key, Object value) { - String[] spec = key.split(";|:", 3); - try { - if (spec.length == 1) { - return new KeyValue(spec[0], value); - } else if (spec.length == 2) { - return new KeyValue(spec[0], getConverter(spec[1]).convert(String.valueOf(value))); - } else if (spec.length == 3) { - return new KeyValue(spec[0], getConverter(spec[1]).convert(String.valueOf(value), spec[2])); - } else { - return new KeyValue(spec[0], value, "Unsupported spec :" + key); - } - } catch (Exception e) { - return new KeyValue(spec[0], value, e.toString()); + Type(IConverter converter, String... aliases) { + this.converter = converter; + this.aliases = Arrays.asList(aliases); } } - static class KeyValue { - - private String key = null; - private Object value = null; - private String grokFailure = null; - - public KeyValue(String key, Object value) { - this.key = key; - this.value = value; - } - - public KeyValue(String key, Object value, String grokFailure) { - this.key = key; - this.value = value; - this.grokFailure = grokFailure; - } - - public boolean hasGrokFailure() { - return grokFailure != null; - } - - public String getGrokFailure() { - return this.grokFailure; - } - - public String getKey() { - return key; - } + private static final Pattern SPLITTER = Pattern.compile("[:;]"); - public void setKey(String key) { - this.key = key; - } + private static final Map TYPES = + Arrays.stream(Type.values()) + .collect(Collectors.toMap(t -> t.name().toLowerCase(), t -> t)); - public Object getValue() { - return value; - } + private static final Map TYPE_ALIASES = + Arrays.stream(Type.values()) + .flatMap(type -> type.aliases.stream().map(alias -> new AbstractMap.SimpleEntry<>(alias, type))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - public void setValue(Object value) { - this.value = value; + private static Type getType(String key) { + key = key.toLowerCase(); + Type type = TYPES.getOrDefault(key, TYPE_ALIASES.get(key)); + if (type == null) { + throw new IllegalArgumentException("Invalid data type :" + key); } + return type; } - - abstract static class IConverter { - - public T convert(String value, String informat) throws Exception { - return null; - } - - public abstract T convert(String value) throws Exception; + public static Map getConverters(Collection groupNames, Object... params) { + return groupNames.stream() + .filter(Converter::containsDelimiter) + .collect(Collectors.toMap(Function.identity(), key -> { + String[] list = splitGrokPattern(key); + IConverter converter = getType(list[1]).converter; + if (list.length == 3) { + converter = converter.newConverter(list[2], params); + } + return converter; + })); } - - static class ByteConverter extends IConverter { - - @Override - public Byte convert(String value) throws Exception { - return Byte.parseByte(value); - } + public static Map getGroupTypes(Collection groupNames) { + return groupNames.stream() + .filter(Converter::containsDelimiter) + .map(Converter::splitGrokPattern) + .collect(Collectors.toMap( + l -> l[0], + l -> getType(l[1]) + )); } - - static class BooleanConverter extends IConverter { - - @Override - public Boolean convert(String value) throws Exception { - return Boolean.parseBoolean(value); - } + public static String extractKey(String key) { + return splitGrokPattern(key)[0]; } - - static class ShortConverter extends IConverter { - - @Override - public Short convert(String value) throws Exception { - return Short.parseShort(value); - } + private static boolean containsDelimiter(String string) { + return string.indexOf(':') >= 0 || string.indexOf(';') >= 0; } - - static class IntegerConverter extends IConverter { - - @Override - public Integer convert(String value) throws Exception { - return Integer.parseInt(value); - } + private static String[] splitGrokPattern(String string) { + return SPLITTER.split(string, 3); } + interface IConverter { - static class LongConverter extends IConverter { + T convert(String value); - @Override - public Long convert(String value) throws Exception { - return Long.parseLong(value); + default IConverter newConverter(String param, Object... params) { + return this; } } - static class FloatConverter extends IConverter { - - @Override - public Float convert(String value) throws Exception { - return Float.parseFloat(value); - } - } - + static class DateConverter implements IConverter { - static class DoubleConverter extends IConverter { + private final DateTimeFormatter formatter; + private final ZoneId timeZone; - @Override - public Double convert(String value) throws Exception { - return Double.parseDouble(value); + public DateConverter() { + this.formatter = DateTimeFormatter.ISO_DATE_TIME; + this.timeZone = ZoneOffset.UTC; } - } - - - static class StringConverter extends IConverter { - @Override - public String convert(String value) throws Exception { - return value; + private DateConverter(DateTimeFormatter formatter, ZoneId timeZone) { + this.formatter = formatter; + this.timeZone = timeZone; } - } - - - static class DateConverter extends IConverter { @Override - public Date convert(String value) throws Exception { - return DateFormat.getDateTimeInstance(DateFormat.SHORT, - DateFormat.SHORT, - Converter.locale).parse(value); + public Instant convert(String value) { + TemporalAccessor dt = formatter + .parseBest(value.trim(), ZonedDateTime::from, LocalDateTime::from, OffsetDateTime::from, Instant::from, + LocalDate::from); + if (dt instanceof ZonedDateTime) { + return ((ZonedDateTime) dt).toInstant(); + } else if (dt instanceof LocalDateTime) { + return ((LocalDateTime) dt).atZone(timeZone).toInstant(); + } else if (dt instanceof OffsetDateTime) { + return ((OffsetDateTime) dt).atZoneSameInstant(timeZone).toInstant(); + } else if (dt instanceof Instant) { + return ((Instant) dt); + } else if (dt instanceof LocalDate) { + return ((LocalDate) dt).atStartOfDay(timeZone).toInstant(); + } else { + return null; + } } @Override - public Date convert(String value, String informat) throws Exception { - SimpleDateFormat formatter = new SimpleDateFormat(informat, Converter.locale); - return formatter.parse(value); + public DateConverter newConverter(String param, Object... params) { + if (!(params.length == 1 && params[0] instanceof ZoneId)) { + throw new IllegalArgumentException("Invalid parameters"); + } + return new DateConverter(DateTimeFormatter.ofPattern(param), (ZoneId) params[0]); } - } - } - diff --git a/src/main/java/io/krakens/grok/api/Discovery.java b/src/main/java/io/krakens/grok/api/Discovery.java index b7f6c20..94a34f7 100644 --- a/src/main/java/io/krakens/grok/api/Discovery.java +++ b/src/main/java/io/krakens/grok/api/Discovery.java @@ -1,24 +1,8 @@ package io.krakens.grok.api; -/******************************************************************************* - * Copyright 2014 Anthony Corbacho and contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -27,10 +11,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import io.krakens.grok.api.exception.GrokException; - import org.apache.commons.lang3.StringUtils; + /** * {@code Discovery} try to find the best pattern for the given string. * @@ -60,10 +43,8 @@ private Map sort(Map groks) { List groky = new ArrayList(groks.values()); Map grokMap = new LinkedHashMap(); Collections.sort(groky, new Comparator() { - public int compare(Grok g1, Grok g2) { - return (this.complexity(g1.getNamedRegex()) < this.complexity(g2.getNamedRegex())) ? 1 - : 0; + return (this.complexity(g1.getNamedRegex()) < this.complexity(g2.getNamedRegex())) ? 1 : 0; } private int complexity(String expandedPattern) { @@ -74,8 +55,8 @@ private int complexity(String expandedPattern) { } }); - for (Grok g : groky) { - grokMap.put(g.getSaved_pattern(), g); + for (Grok grok : groky) { + grokMap.put(grok.getSaved_pattern(), grok); } return grokMap; @@ -111,24 +92,21 @@ public String discover(String text) { Map grokPatterns = grok.getPatterns(); // Boolean done = false; String texte = text; + GrokCompiler compiler = GrokCompiler.newInstance(); + compiler.register(grokPatterns); // Compile the pattern - Iterator> it = grokPatterns.entrySet().iterator(); - while (it.hasNext()) { + for (Entry stringStringEntry : grokPatterns.entrySet()) { @SuppressWarnings("rawtypes") - Map.Entry pairs = (Map.Entry) it.next(); + Entry pairs = (Entry) stringStringEntry; String key = pairs.getKey().toString(); - Grok grok = new Grok(); - // g.patterns.putAll( gPatterns ); try { - grok.copyPatterns(grokPatterns); + Grok grok = compiler.compile("%{" + key + "}"); grok.setSaved_pattern(key); - grok.compile("%{" + key + "}"); groks.put(key, grok); - } catch (GrokException e) { + } catch (Exception e) { // Add logger - continue; } } @@ -138,12 +116,9 @@ public String discover(String text) { // while (!done){ // done = true; - Iterator> pit = patterns.entrySet().iterator(); - while (pit.hasNext()) { - @SuppressWarnings("rawtypes") - Map.Entry pairs = (Map.Entry) pit.next(); - String key = pairs.getKey().toString(); - Grok value = (Grok) pairs.getValue(); + for (Entry pairs : patterns.entrySet()) { + String key = pairs.getKey(); + Grok value = pairs.getValue(); // We want to search with more complex pattern // We avoid word, small number, space.... diff --git a/src/main/java/io/krakens/grok/api/Garbage.java b/src/main/java/io/krakens/grok/api/Garbage.java deleted file mode 100644 index 16b3c77..0000000 --- a/src/main/java/io/krakens/grok/api/Garbage.java +++ /dev/null @@ -1,149 +0,0 @@ -package io.krakens.grok.api; -/******************************************************************************* - * Copyright 2014 Anthony Corbacho and contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -/** - * The Leon the professional of {@code Grok}.
- * Garbage is use by grok to remove or rename elements before getting the final output - * - * @since 0.0.2 - */ -public class Garbage { - - private List toRemove; - private Map toRename; - - /** - * Create a new {@code Garbage} object. - */ - public Garbage() { - - toRemove = new ArrayList(); - toRename = new TreeMap(); - /** this is a default value to remove */ - toRemove.add("UNWANTED"); - } - - /** - * Set a new name to be change when exporting the final output. - * - * @param origin : original field name - * @param value : New field name to apply - */ - public void addToRename(String origin, Object value) { - if (origin == null || value == null) { - return; - } - - if (!origin.isEmpty() && !value.toString().isEmpty()) { - toRename.put(origin, value); - } - } - - /** - * Set a field to be remove when exporting the final output. - * - * @param name of the field to remove - */ - public void addToRemove(String name) { - if (name == null) { - return; - } - - if (!name.isEmpty()) { - toRemove.add(name); - } - } - - /** - * Set a list of field name to be remove when exporting the final output. - * - * @param lst list of elem to remove - */ - public void addToRemove(List lst) { - if (lst == null) { - return; - } - - if (!lst.isEmpty()) { - toRemove.addAll(lst); - } - } - - /** - * Remove from the map the unwilling items. - * - * @param map to clean - * @return nb of deleted item - */ - public int remove(Map map) { - int item = 0; - - if (map == null) { - return item; - } - - if (map.isEmpty()) { - return item; - } - - for (Iterator> it = map.entrySet().iterator(); it.hasNext(); ) { - Map.Entry entry = it.next(); - for (int i = 0; i < toRemove.size(); i++) { - if (entry.getKey().equals(toRemove.get(i))) { - it.remove(); - item++; - } - } - } - return item; - } - - /** - * Rename the item from the map. - * - * @param map elem to rename - * @return nb of renamed items - */ - public int rename(Map map) { - int item = 0; - - if (map == null) { - return item; - } - - if (map.isEmpty() || toRename.isEmpty()) { - return item; - } - - for (Iterator> it = toRename.entrySet().iterator(); it.hasNext(); ) { - Map.Entry entry = it.next(); - if (map.containsKey(entry.getKey())) { - Object obj = map.remove(entry.getKey()); - map.put(entry.getValue().toString(), obj); - item++; - } - } - return item; - } - -} diff --git a/src/main/java/io/krakens/grok/api/Grok.java b/src/main/java/io/krakens/grok/api/Grok.java index 2b93b7e..aa7798d 100644 --- a/src/main/java/io/krakens/grok/api/Grok.java +++ b/src/main/java/io/krakens/grok/api/Grok.java @@ -1,121 +1,76 @@ package io.krakens.grok.api; -/******************************************************************************* - * Copyright 2014 Anthony Corbacho and contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -import static java.lang.String.format; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.Serializable; +import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.TreeMap; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import io.krakens.grok.api.exception.GrokException; - +import io.krakens.grok.api.Converter.IConverter; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * {@code Grok} parse arbitrary text and structure it. *
* {@code Grok} is simple API that allows you to easily parse logs * and other files (single line). With {@code Grok}, - * you can turn unstructured log and event data into structured data (JSON). - *
- * example:
- * 
- *  Grok grok = Grok.create("patterns/patterns");
- *  grok.compile("%{USER}");
- *  Match gm = grok.match("root");
- *  gm.captures();
- *
+ * you can turn unstructured log and event data into structured data. * * @since 0.0.1 */ -public class Grok implements Serializable { - - private static final Logger LOG = LoggerFactory.getLogger(Grok.class); +public class Grok { /** * Named regex of the originalGrokPattern. */ - private String namedRegex; + private final String namedRegex; /** * Map of the named regex of the originalGrokPattern * with id = namedregexid and value = namedregex. */ - private Map namedRegexCollection; + private final Map namedRegexCollection; /** * Original {@code Grok} pattern (expl: %{IP}). */ - private String originalGrokPattern; + private final String originalGrokPattern; /** * Pattern of the namedRegex. */ - private Pattern compiledNamedRegex; - /** - * {@code Grok} discovery. - */ - private Discovery disco; + private final Pattern compiledNamedRegex; + /** * {@code Grok} patterns definition. */ - private Map grokPatternDefinition; + private final Map grokPatternDefinition; - /** - * only use in grok discovery. - */ - private String savedPattern; + public final Set namedGroups; - /** - * ignore empty captures. - */ - private boolean keepEmptyCaptures = true; + public final Map groupTypes; - /** - * automatic conversion of values. - */ - private boolean automaticConversionEnabled = true; + public final Map converters; /** - * Create Empty {@code Grok}. + * {@code Grok} discovery. */ - public static final Grok EMPTY = new Grok(); + private Discovery disco; - /** - * Create a new empty{@code Grok} object. - */ - public Grok() { - originalGrokPattern = StringUtils.EMPTY; - disco = null; - namedRegex = StringUtils.EMPTY; - compiledNamedRegex = null; - grokPatternDefinition = new TreeMap(); - namedRegexCollection = new TreeMap(); - savedPattern = StringUtils.EMPTY; + /** only use in grok discovery. */ + private String savedPattern = ""; + + public Grok(String pattern, + String namedRegex, + Map namedRegexCollection, + Map patternDefinitions, + ZoneId defaultTimeZone) { + this.originalGrokPattern = pattern; + this.namedRegex = namedRegex; + this.compiledNamedRegex = Pattern.compile(namedRegex); + this.namedRegexCollection = namedRegexCollection; + this.namedGroups = GrokUtils.getNameGroups(namedRegex); + this.groupTypes = Converter.getGroupTypes(namedRegexCollection.values()); + this.converters = Converter.getConverters(namedRegexCollection.values(), defaultTimeZone); + this.grokPatternDefinition = patternDefinitions; } public String getSaved_pattern() { @@ -126,87 +81,6 @@ public void setSaved_pattern(String savedpattern) { this.savedPattern = savedpattern; } - /** - * Create a {@code Grok} instance with the given patterns file and - * a {@code Grok} pattern. - * - * @param grokPatternPath Path to the pattern file - * @param grokExpression - OPTIONAL - Grok pattern to compile ex: %{APACHELOG} - * @return {@code Grok} instance - * @throws GrokException runtime expt - */ - public static Grok create(String grokPatternPath, String grokExpression) - throws GrokException { - if (StringUtils.isBlank(grokPatternPath)) { - throw new GrokException("{grokPatternPath} should not be empty or null"); - } - Grok grok = new Grok(); - grok.addPatternFromFile(grokPatternPath); - if (StringUtils.isNotBlank(grokExpression)) { - grok.compile(grokExpression, false); - } - return grok; - } - - /** - * Create a {@code Grok} instance with included default patterns. - * - * @return {@code Grok} instance - */ - public static Grok create() throws GrokException { - Grok grok = new Grok(); - grok.addPatternFromClasspath("/patterns/patterns"); - return grok; - } - - /** - * Create a {@code Grok} instance with the given grok patterns file. - * - * @param grokPatternPath : Path to the pattern file - * @return Grok - * @throws GrokException runtime expt - */ - public static Grok create(String grokPatternPath) throws GrokException { - return create(grokPatternPath, null); - } - - /** - * Add custom pattern to grok in the runtime. - * - * @param name : Pattern Name - * @param pattern : Regular expression Or {@code Grok} pattern - * @throws GrokException runtime expt - **/ - public void addPattern(String name, String pattern) throws GrokException { - if (StringUtils.isBlank(name)) { - throw new GrokException("Invalid Pattern name"); - } - if (StringUtils.isBlank(pattern)) { - throw new GrokException("Invalid Pattern"); - } - grokPatternDefinition.put(name, pattern); - } - - /** - * Copy the given Map of patterns (pattern name, regular expression) to {@code Grok}, - * duplicate element will be override. - * - * @param cpy : Map to copy - * @throws GrokException runtime expt - **/ - public void copyPatterns(Map cpy) throws GrokException { - if (cpy == null) { - throw new GrokException("Invalid Patterns"); - } - - if (cpy.isEmpty()) { - throw new GrokException("Invalid Patterns"); - } - for (Map.Entry entry : cpy.entrySet()) { - grokPatternDefinition.put(entry.getKey().toString(), entry.getValue().toString()); - } - } - /** * Get the current map of {@code Grok} pattern. * @@ -218,8 +92,6 @@ public Map getPatterns() { /** * Get the named regex from the {@code Grok} pattern.
- * See {@link #compile(String)} for more detail. - * * @return named regex */ public String getNamedRegex() { @@ -227,82 +99,31 @@ public String getNamedRegex() { } /** - * Add patterns to {@code Grok} from the given file. + * Original grok pattern used to compile to the named regex. * - * @param path : Path of the grok pattern - * @throws GrokException runtime expt + * @return String Original Grok pattern */ - public void addPatternFromFile(String path) throws GrokException { - - File file = new File(path); - if (!file.exists()) { - throw new GrokException("Pattern file " + file.getAbsolutePath() + " not found"); - } - - if (!file.canRead()) { - throw new GrokException("Pattern file " + file.getAbsolutePath() + " cannot be read"); - } - - try (FileReader r = new FileReader(file)) { - addPatternFromReader(r); - } catch (IOException e) { - throw new GrokException(e.getMessage(), e); - } - } - - public void addPatternFromClasspath(String path) throws GrokException { - final InputStream inputStream = this.getClass().getResourceAsStream(path); - try (Reader r = new InputStreamReader(inputStream)) { - addPatternFromReader(r); - } catch (IOException e) { - throw new GrokException(e.getMessage(), e); - } + public String getOriginalGrokPattern() { + return originalGrokPattern; } /** - * Add patterns to {@code Grok} from a Reader. + * Get the named regex from the given id. * - * @param reader : Reader with {@code Grok} patterns - * @throws GrokException runtime expt - */ - public void addPatternFromReader(Reader reader) throws GrokException { - BufferedReader br = new BufferedReader(reader); - String line; - // We dont want \n and commented line - Pattern pattern = Pattern.compile("^([A-z0-9_]+)\\s+(.*)$"); - try { - while ((line = br.readLine()) != null) { - Matcher matcher = pattern.matcher(line); - if (matcher.matches()) { - this.addPattern(matcher.group(1), matcher.group(2)); - } - } - br.close(); - } catch (IOException | GrokException e) { - throw new GrokException(e.getMessage(), e); - } - } - - /** - * Ignore empty captures. + * @param id : named regex id + * @return String of the named regex */ - public void setKeepEmptyCaptures(boolean ignore) { - this.keepEmptyCaptures = ignore; - } - - public boolean isKeepEmptyCaptures() { - return this.keepEmptyCaptures; + public String getNamedRegexCollectionById(String id) { + return namedRegexCollection.get(id); } /** - * Disable automatic conversion of values. + * Get the full collection of the named regex. + * + * @return named RegexCollection */ - public void disableAutomaticConversion() { - this.automaticConversionEnabled = false; - } - - public boolean isAutomaticConversionEnabled() { - return automaticConversionEnabled; + public Map getNamedRegexCollection() { + return namedRegexCollection; } /** @@ -310,12 +131,11 @@ public boolean isAutomaticConversionEnabled() { * And return the json representation of the matched element * * @param log : log to match - * @return json representation og the log + * @return map containing matches */ - public String capture(String log) { + public Map capture(String log) { Match match = match(log); - match.captures(); - return match.toJson(); + return match.capture(); } /** @@ -323,14 +143,12 @@ public String capture(String log) { * and return the list of json representation of the matched elements. * * @param logs : list of log - * @return list of json representation of the log + * @return list of maps containing matches */ - public List captures(List logs) { - List matched = new ArrayList(); + public ArrayList> capture(List logs) { + final ArrayList> matched = new ArrayList<>(); for (String log : logs) { - Match match = match(log); - match.captures(); - matched.add(match.toJson()); + matched.add(capture(log)); } return matched; } @@ -342,102 +160,19 @@ public List captures(List logs) { * @param text : Single line of log * @return Grok Match */ - public Match match(String text) { + public Match match(CharSequence text) { if (compiledNamedRegex == null || StringUtils.isBlank(text)) { return Match.EMPTY; } Matcher matcher = compiledNamedRegex.matcher(text); - Match match = new Match(); if (matcher.find()) { - match.setSubject(text); - match.setGrok(this); - match.setMatch(matcher); - match.setStart(matcher.start(0)); - match.setEnd(matcher.end(0)); - } - return match; - } - - /** - * Compile the {@code Grok} pattern to named regex pattern. - * - * @param pattern : Grok pattern (ex: %{IP}) - * @throws GrokException runtime expt - */ - public void compile(String pattern) throws GrokException { - compile(pattern, false); - } - - /** - * Compile the {@code Grok} pattern to named regex pattern. - * - * @param pattern : Grok pattern (ex: %{IP}) - * @param namedOnly : Whether to capture named expressions only or not (i.e. %{IP:ip} but not ${IP}) - * @throws GrokException runtime expt - */ - public void compile(String pattern, boolean namedOnly) throws GrokException { - - if (StringUtils.isBlank(pattern)) { - throw new GrokException("{pattern} should not be empty or null"); + return new Match( + text, this, matcher, matcher.start(0), matcher.end(0) + ); } - namedRegex = pattern; - originalGrokPattern = pattern; - int index = 0; - /** flag for infinite recursion. */ - int iterationLeft = 1000; - Boolean continueIteration = true; - - // Replace %{foo} with the regex (mostly groupname regex) - // and then compile the regex - while (continueIteration) { - continueIteration = false; - if (iterationLeft <= 0) { - throw new GrokException("Deep recursion pattern compilation of " + originalGrokPattern); - } - iterationLeft--; - - Matcher matcher = GrokUtils.GROK_PATTERN.matcher(namedRegex); - // Match %{Foo:bar} -> pattern name and subname - // Match %{Foo=regex} -> add new regex definition - if (matcher.find()) { - continueIteration = true; - Map group = GrokUtils.namedGroups(matcher, matcher.group()); - if (group.get("definition") != null) { - try { - addPattern(group.get("pattern"), group.get("definition")); - group.put("name", group.get("name") + "=" + group.get("definition")); - } catch (GrokException e) { - throw new RuntimeException(e); - } - } - int count = StringUtils.countMatches(namedRegex, "%{" + group.get("name") + "}"); - for (int i = 0; i < count; i++) { - String definitionOfPattern = grokPatternDefinition.get(group.get("pattern")); - if (definitionOfPattern == null) { - throw new GrokException(format("No definition for key '%s' found, aborting", - group.get("pattern"))); - } - String replacement = String.format("(?%s)", index, definitionOfPattern); - if (namedOnly && group.get("subname") == null) { - replacement = String.format("(?:%s)", definitionOfPattern); - } - namedRegexCollection.put("name" + index, - (group.get("subname") != null ? group.get("subname") : group.get("name"))); - namedRegex = - StringUtils.replace(namedRegex, "%{" + group.get("name") + "}", replacement, 1); - // System.out.println(_expanded_pattern); - index++; - } - } - } - - if (namedRegex.isEmpty()) { - throw new GrokException("Pattern not fount"); - } - // Compile the regex - compiledNamedRegex = Pattern.compile(namedRegex); + return Match.EMPTY; } /** @@ -454,32 +189,4 @@ public String discover(String input) { } return disco.discover(input); } - - /** - * Original grok pattern used to compile to the named regex. - * - * @return String Original Grok pattern - */ - public String getOriginalGrokPattern() { - return originalGrokPattern; - } - - /** - * Get the named regex from the given id. - * - * @param id : named regex id - * @return String of the named regex - */ - public String getNamedRegexCollectionById(String id) { - return namedRegexCollection.get(id); - } - - /** - * Get the full collection of the named regex. - * - * @return named RegexCollection - */ - public Map getNamedRegexCollection() { - return namedRegexCollection; - } } diff --git a/src/main/java/io/krakens/grok/api/GrokCompiler.java b/src/main/java/io/krakens/grok/api/GrokCompiler.java new file mode 100644 index 0000000..9d13d13 --- /dev/null +++ b/src/main/java/io/krakens/grok/api/GrokCompiler.java @@ -0,0 +1,180 @@ +package io.krakens.grok.api; + +import static java.lang.String.format; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import io.krakens.grok.api.exception.GrokException; + +import org.apache.commons.lang3.StringUtils; + +public class GrokCompiler { + /** + * {@code Grok} patterns definitions. + */ + private final Map grokPatternDefinitions = new HashMap<>(); + + private GrokCompiler() {} + + public static GrokCompiler newInstance() { + return new GrokCompiler(); + } + + public Map getPatternDefinitions() { + return grokPatternDefinitions; + } + + /** + * Registers a new pattern definition. + * + * @param name : Pattern Name + * @param pattern : Regular expression Or {@code Grok} pattern + * @throws GrokException runtime expt + **/ + public void register(String name, String pattern) { + name = Objects.requireNonNull(name).trim(); + pattern = Objects.requireNonNull(pattern).trim(); + + if (!name.isEmpty() && !pattern.isEmpty()) { + grokPatternDefinitions.put(name, pattern); + } + } + + /** + * Registers multiple pattern definitions. + */ + public void register(Map patternDefinitions) { + Objects.requireNonNull(patternDefinitions); + patternDefinitions.forEach(this::register); + } + + public void registerDefaultPatterns() { + registerPatternFromClasspath("/patterns/patterns"); + } + + public void registerPatternFromClasspath(String path) throws GrokException { + final InputStream inputStream = this.getClass().getResourceAsStream(path); + try (Reader reader = new InputStreamReader(inputStream)) { + register(inputStream); + } catch (IOException e) { + throw new GrokException(e.getMessage(), e); + } + } + + /** + * Registers multiple pattern definitions from a given inputStream. + */ + public void register(InputStream input) throws IOException { + // We don't want \n and commented line + Pattern pattern = Pattern.compile("^([A-z0-9_]+)\\s+(.*)$"); + + try ( + BufferedReader in = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8))) { + in.lines() + .map(pattern::matcher) + .filter(Matcher::matches) + .forEach(m -> register(m.group(1), m.group(2))); + } + } + + /** + * Compiles a given Grok pattern and returns a Grok object which can parse the pattern. + */ + public Grok compile(String pattern) { + return compile(pattern, false); + } + + public Grok compile(final String pattern, boolean namedOnly) { + return compile(pattern, ZoneOffset.UTC, namedOnly); + } + + /** + * Compiles a given Grok pattern and returns a Grok object which can parse the pattern. + * + * @param pattern : Grok pattern (ex: %{IP}) + * @param defaultTimeZone : time zone used to parse a timestamp when it doesn't contain the time zone + * @param namedOnly : Whether to capture named expressions only or not (i.e. %{IP:ip} but not ${IP}) + */ + public Grok compile(final String pattern, ZoneId defaultTimeZone, boolean namedOnly) { + + if (StringUtils.isBlank(pattern)) { + throw new IllegalArgumentException("{pattern} should not be empty or null"); + } + + String namedRegex = pattern; + int index = 0; + /** flag for infinite recursion. */ + int iterationLeft = 1000; + Boolean continueIteration = true; + Map patternDefinitions = new HashMap<>(grokPatternDefinitions); + + // output + Map namedRegexCollection = new HashMap<>(); + + // Replace %{foo} with the regex (mostly group name regex) + // and then compile the regex + while (continueIteration) { + continueIteration = false; + if (iterationLeft <= 0) { + throw new IllegalArgumentException("Deep recursion pattern compilation of " + pattern); + } + iterationLeft--; + + Set namedGroups = GrokUtils.getNameGroups(GrokUtils.GROK_PATTERN.pattern()); + Matcher matcher = GrokUtils.GROK_PATTERN.matcher(namedRegex); + // Match %{Foo:bar} -> pattern name and subname + // Match %{Foo=regex} -> add new regex definition + if (matcher.find()) { + continueIteration = true; + Map group = GrokUtils.namedGroups(matcher, namedGroups); + if (group.get("definition") != null) { + patternDefinitions.put(group.get("pattern"), group.get("definition")); + group.put("name", group.get("name") + "=" + group.get("definition")); + } + int count = StringUtils.countMatches(namedRegex, "%{" + group.get("name") + "}"); + for (int i = 0; i < count; i++) { + String definitionOfPattern = patternDefinitions.get(group.get("pattern")); + if (definitionOfPattern == null) { + throw new IllegalArgumentException(format("No definition for key '%s' found, aborting", + group.get("pattern"))); + } + String replacement = String.format("(?%s)", index, definitionOfPattern); + if (namedOnly && group.get("subname") == null) { + replacement = String.format("(?:%s)", definitionOfPattern); + } + namedRegexCollection.put("name" + index, + (group.get("subname") != null ? group.get("subname") : group.get("name"))); + namedRegex = + StringUtils.replace(namedRegex, "%{" + group.get("name") + "}", replacement,1); + // System.out.println(_expanded_pattern); + index++; + } + } + } + + if (namedRegex.isEmpty()) { + throw new IllegalArgumentException("Pattern not found"); + } + + return new Grok( + pattern, + namedRegex, + namedRegexCollection, + patternDefinitions, + defaultTimeZone + ); + } +} diff --git a/src/main/java/io/krakens/grok/api/GrokUtils.java b/src/main/java/io/krakens/grok/api/GrokUtils.java index 3006ad3..f3d71eb 100644 --- a/src/main/java/io/krakens/grok/api/GrokUtils.java +++ b/src/main/java/io/krakens/grok/api/GrokUtils.java @@ -22,7 +22,7 @@ public class GrokUtils { "%\\{" + "(?" + "(?[A-z0-9]+)" - + "(?::(?[A-z0-9_:;\\/\\s\\.]+))?" + + "(?::(?[A-z0-9_:;\\-\\/\\s\\.']+))?" + ")" + "(?:=(?" + "(?:" @@ -32,27 +32,23 @@ public class GrokUtils { + ")?" + "\\}"); - public static final Pattern NAMED_REGEX = Pattern.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>"); + public static final Pattern NAMED_REGEX = Pattern + .compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>"); - private static Set getNameGroups(String regex) { - Set namedGroups = new LinkedHashSet(); - Matcher match = NAMED_REGEX.matcher(regex); - while (match.find()) { - namedGroups.add(match.group(1)); + public static Set getNameGroups(String regex) { + Set namedGroups = new LinkedHashSet<>(); + Matcher matcher = NAMED_REGEX.matcher(regex); + while (matcher.find()) { + namedGroups.add(matcher.group(1)); } return namedGroups; } - public static Map namedGroups(Matcher matcher, - String namedRegex) { - Set groupNames = getNameGroups(matcher.pattern().pattern()); - Matcher localMatcher = matcher.pattern().matcher(namedRegex); - Map namedGroups = new LinkedHashMap(); - if (localMatcher.find()) { - for (String groupName : groupNames) { - String groupValue = localMatcher.group(groupName); - namedGroups.put(groupName, groupValue); - } + public static Map namedGroups(Matcher matcher, Set groupNames) { + Map namedGroups = new LinkedHashMap<>(); + for (String groupName : groupNames) { + String groupValue = matcher.group(groupName); + namedGroups.put(groupName, groupValue); } return namedGroups; } diff --git a/src/main/java/io/krakens/grok/api/Match.java b/src/main/java/io/krakens/grok/api/Match.java index d57d783..8285a78 100644 --- a/src/main/java/io/krakens/grok/api/Match.java +++ b/src/main/java/io/krakens/grok/api/Match.java @@ -1,34 +1,16 @@ package io.krakens.grok.api; -/******************************************************************************* - * Copyright 2014 Anthony Corbacho and contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ + import static java.lang.String.format; import java.util.ArrayList; -import java.util.Iterator; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeMap; import java.util.regex.Matcher; -import io.krakens.grok.api.Converter.KeyValue; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import io.krakens.grok.api.Converter.IConverter; /** * {@code Match} is a representation in {@code Grok} world of your log. @@ -36,99 +18,48 @@ * @since 0.0.1 */ public class Match { + private final CharSequence subject; + private final Grok grok; + private final Matcher match; + private final int start; + private final int end; - // Create Empty grok matcher. - public static final Match EMPTY = new Match(); - - private static final Gson PRETTY_GSON = new GsonBuilder().setPrettyPrinting().create(); - private static final Gson GSON = new GsonBuilder().create(); - - private String subject; - private Map capture; - private Garbage garbage; - private Grok grok; - private Matcher match; - private int start; - private int end; - - /** - * For thread safety. - */ - private static ThreadLocal matchHolder = ThreadLocal.withInitial(() -> new Match()); + private Map capture = Collections.emptyMap(); /** * Create a new {@code Match} object. */ - public Match() { - subject = "Nothing"; - grok = null; - match = null; - capture = new TreeMap(); - garbage = new Garbage(); - start = 0; - end = 0; + public Match(CharSequence subject, Grok grok, Matcher match, int start, int end) { + this.subject = subject; + this.grok = grok; + this.match = match; + this.start = start; + this.end = end; } - public void setGrok(Grok grok) { - if (grok != null) { - this.grok = grok; - } - } + /** + * Create Empty grok matcher. + */ + public static final Match EMPTY = new Match("", null, null, 0, 0); public Matcher getMatch() { return match; } - public void setMatch(Matcher match) { - this.match = match; - } - public int getStart() { return start; } - public void setStart(int start) { - this.start = start; - } - public int getEnd() { return end; } - public void setEnd(int end) { - this.end = end; - } - - /** - * Singleton. - * - * @return instance of Match - */ - public static Match getInstance() { - return matchHolder.get(); - } - - /** - * Set the single line of log to parse. - * - * @param text : single line of log - */ - public void setSubject(String text) { - if (text == null) { - return; - } - if (text.isEmpty()) { - return; - } - subject = text; - } - /** * Retrurn the single line of log. * * @return the single line of log */ - public String getSubject() { + public CharSequence getSubject() { return subject; } @@ -136,14 +67,14 @@ public String getSubject() { * Match to the subject the regex and save the matched element into a map. * * Multiple values for the same key are stored as list. + * */ - public void captures() { - capture(false); - + public Map capture() { + return capture(false); } /** - * Match to the subject the regex and save the matched element into a map. + * Match to the subject the regex and save the matched element into a map * * Multiple values to the same key are flattened to one value: the sole non-null value will be captured. * Should there be multiple non-null values a RuntimeException is being thrown. @@ -151,60 +82,57 @@ public void captures() { * This can be used in cases like: (foo (.*:message) bar|bar (.*:message) foo) where the regexp guarantees that only * one value will be captured. * - * See also {@link #captures} which returns multiple values of the same key as list. + * See also {@link #capture} which returns multiple values of the same key as list. + * */ - public void capturesFlattened() { - capture(true); + public Map captureFlattened() { + return capture(true); } - @SuppressWarnings("unchecked") - private void capture(boolean flattened) { + private Map capture(boolean flattened ) { if (match == null) { - return; + return Collections.emptyMap(); } - capture.clear(); - boolean automaticConversionEnabled = grok.isAutomaticConversionEnabled(); + + if (!capture.isEmpty()) { + return capture; + } + + capture = new HashMap<>(); // _capture.put("LINE", this.line); // _capture.put("LENGTH", this.line.length() +""); - Map mappedw = GrokUtils.namedGroups(this.match, this.subject); - Iterator> it = mappedw.entrySet().iterator(); - while (it.hasNext()) { - - @SuppressWarnings("rawtypes") - Map.Entry pairs = (Map.Entry) it.next(); - String key = null; - Object value = null; - if (this.grok.getNamedRegexCollectionById(pairs.getKey().toString()) == null) { - key = pairs.getKey().toString(); - } else if (!this.grok.getNamedRegexCollectionById(pairs.getKey().toString()).isEmpty()) { - key = this.grok.getNamedRegexCollectionById(pairs.getKey().toString()); + Map mappedw = GrokUtils.namedGroups(this.match, this.grok.namedGroups); + + mappedw.forEach((key, valueString) -> { + String id = this.grok.getNamedRegexCollectionById(key); + if (id != null && !id.isEmpty()) { + key = id; } - if (pairs.getValue() != null) { - value = pairs.getValue().toString(); - if (automaticConversionEnabled) { - KeyValue keyValue = Converter.convert(key, value); + if ("UNWANTED".equals(key)) { + return; + } - // get validated key - key = keyValue.getKey(); + Object value = valueString; + if (valueString != null) { + IConverter converter = grok.converters.get(key); - // resolve value - if (keyValue.getValue() instanceof String) { - value = cleanString((String) keyValue.getValue()); - } else { - value = keyValue.getValue(); + if (converter != null) { + key = Converter.extractKey(key); + try { + value = converter.convert(valueString); + } catch (Exception e) { + capture.put(key + "_grokfailure", e.toString()); } - // set if grok failure - if (keyValue.hasGrokFailure()) { - capture.put(key + "_grokfailure", keyValue.getGrokFailure()); + if (value instanceof String) { + value = cleanString((String) value); } + } else { + value = cleanString(valueString); } - } else if (!grok.isKeepEmptyCaptures()) { - it.remove(); - continue; } if (capture.containsKey(key)) { @@ -216,8 +144,7 @@ private void capture(boolean flattened) { } if (currentValue != null && value != null) { throw new RuntimeException( - format( - "key '%s' has multiple non-null values, this is not allowed in flattened mode, values:'%s', '%s'", + format("key '%s' has multiple non-null values, this is not allowed in flattened mode, values:'%s', '%s'", key, currentValue, value)); @@ -235,9 +162,11 @@ private void capture(boolean flattened) { } else { capture.put(key, value); } + }); - it.remove(); // avoids a ConcurrentModificationException - } + capture = Collections.unmodifiableMap(capture); + + return capture; } /** @@ -247,78 +176,22 @@ private void capture(boolean flattened) { * @return unquoted string: my/text */ private String cleanString(String value) { - if (value == null) { - return null; - } - if (value.isEmpty()) { + if (value == null || value.isEmpty()) { return value; } - char[] tmp = value.toCharArray(); - if (tmp.length == 1 && (tmp[0] == '"' || tmp[0] == '\'')) { - value = "";//empty string - } else if ((tmp[0] == '"' && tmp[value.length() - 1] == '"') - || (tmp[0] == '\'' && tmp[value.length() - 1] == '\'')) { - value = value.substring(1, value.length() - 1); - } - return value; - } + char firstChar = value.charAt(0); + char lastChar = value.charAt(value.length() - 1); - /** - * Get the json representation of the matched element. - *

- * example: map [ {IP: 127.0.0.1}, {status:200}] will return {"IP":"127.0.0.1", "status":200} - *

- * If pretty is set to true, json will return prettyprint json string. - * - * @return Json of the matched element in the text - */ - public String toJson(Boolean pretty) { - if (capture == null) { - return "{}"; - } - if (capture.isEmpty()) { - return "{}"; - } - - this.cleanMap(); - Gson gs; - if (pretty) { - gs = PRETTY_GSON; - } else { - gs = GSON; + if (firstChar == lastChar && (firstChar == '"' || firstChar == '\'')) { + if (value.length() == 1) { + return ""; + } else { + return value.substring(1, value.length() - 1); + } } - return gs.toJson(/* cleanMap( */capture/* ) */); - } - - /** - * Get the json representation of the matched element. - *

- * example: map [ {IP: 127.0.0.1}, {status:200}] will return {"IP":"127.0.0.1", "status":200} - *

- * - * @return Json of the matched element in the text - */ - public String toJson() { - return toJson(false); - } - /** - * Get the map representation of the matched element in the text. - * - * @return map object from the matched element in the text - */ - public Map toMap() { - this.cleanMap(); - return capture; - } - - /** - * Remove and rename the unwanted elelents in the matched map. - */ - private void cleanMap() { - garbage.rename(capture); - garbage.remove(capture); + return value; } /** diff --git a/src/main/java/io/krakens/grok/api/exception/GrokError.java b/src/main/java/io/krakens/grok/api/exception/GrokError.java deleted file mode 100644 index 07feade..0000000 --- a/src/main/java/io/krakens/grok/api/exception/GrokError.java +++ /dev/null @@ -1,32 +0,0 @@ -package io.krakens.grok.api.exception; -/******************************************************************************* - * Copyright 2014 Anthony Corbacho and contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ - -/** - * Not used, too much c style.. - * - */ -@Deprecated -public class GrokError { - - public static final int GROK_OK = 0; - public static final int GROK_ERROR_FILE_NOT_ACCESSIBLE = 1; - public static final int GROK_ERROR_PATTERN_NOT_FOUND = 2; - public static final int GROK_ERROR_UNEXPECTED_READ_SIZE = 3; - public static final int GROK_ERROR_COMPILE_FAILED = 4; - public static final int GROK_ERROR_UNINITIALIZED = 5; - public static final int GROK_ERROR_NOMATCH = 6; -} diff --git a/src/main/java/io/krakens/grok/api/exception/GrokException.java b/src/main/java/io/krakens/grok/api/exception/GrokException.java index 2b67dad..30b2d83 100644 --- a/src/main/java/io/krakens/grok/api/exception/GrokException.java +++ b/src/main/java/io/krakens/grok/api/exception/GrokException.java @@ -1,19 +1,4 @@ package io.krakens.grok.api.exception; -/******************************************************************************* - * Copyright 2014 Anthony Corbacho and contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ /** * Signals that an {@code Grok} exception of some sort has occurred. @@ -22,7 +7,7 @@ * * @since 0.0.4 */ -public class GrokException extends Exception { +public class GrokException extends RuntimeException { private static final long serialVersionUID = 1L; diff --git a/src/test/java/io/krakens/grok/api/ApacheDataTypeTest.java b/src/test/java/io/krakens/grok/api/ApacheDataTypeTest.java index 5a52d53..e26834e 100644 --- a/src/test/java/io/krakens/grok/api/ApacheDataTypeTest.java +++ b/src/test/java/io/krakens/grok/api/ApacheDataTypeTest.java @@ -1,77 +1,84 @@ package io.krakens.grok.api; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; +import static org.junit.Assert.assertEquals; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.Locale; import java.util.Map; import io.krakens.grok.api.exception.GrokException; +import com.google.common.io.Resources; +import org.assertj.core.api.Assertions; +import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; - @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class ApacheDataTypeTest { - private final String line = "64.242.88.10 - - [07/Mar/2004:16:45:56 -0800] \"GET /twiki/bin/attach/Main/" - + "PostfixCommands HTTP/1.1\" 401 12846"; - static { - Locale.setDefault(Locale.ENGLISH); + Locale.setDefault(Locale.ROOT); + } + + private final String line = + "64.242.88.10 - - [07/Mar/2004:16:45:56 -0800] \"GET /twiki/bin/attach/Main/PostfixCommands HTTP/1.1\" 401 12846"; + + private GrokCompiler compiler; + + @Before + public void setup() throws Exception { + compiler = GrokCompiler.newInstance(); + compiler.register(Resources.getResource(ResourceManager.PATTERNS).openStream()); } @Test - public void test002_httpd_access_semi() throws GrokException, IOException, ParseException { - Grok grok = Grok.create(ResourceManager.PATTERNS, - "%{IPORHOST:clientip} %{USER:ident;boolean} %{USER:auth} \\[%{HTTPDATE:timestamp;date;dd/MMM" - + "/yyyy:HH:mm:ss Z}\\] \"(?:%{WORD:verb;string} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion;float})?" - + "|%{DATA:rawrequest})\" %{NUMBER:response;int} (?:%{NUMBER:bytes;long}|-)"); + public void test002_httpd_access_semi() throws GrokException { + Grok grok = compiler.compile( + "%{IPORHOST:clientip} %{USER:ident;boolean} %{USER:auth} " + + "\\[%{HTTPDATE:timestamp;date;dd/MMM/yyyy:HH:mm:ss Z}\\] \"(?:%{WORD:verb;string} %{NOTSPACE:request}" + + "(?: HTTP/%{NUMBER:httpversion;float})?|%{DATA:rawrequest})\" %{NUMBER:response;int} " + + "(?:%{NUMBER:bytes;long}|-)"); System.out.println(line); Match gm = grok.match(line); - gm.captures(); - - assertNotEquals("{\"Error\":\"Error\"}", gm.toJson()); + Map map = gm.capture(); - Map map = gm.toMap(); - assertTrue(map.get("timestamp") - .equals(new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z").parse("07/Mar/2004:16:45:56 -0800"))); - assertTrue(map.get("response").equals(Integer.valueOf(401))); - assertTrue(map.get("ident").equals(Boolean.FALSE)); - assertTrue(map.get("httpversion").equals(Float.valueOf(1.1f))); - assertTrue(map.get("bytes").equals(Long.valueOf(12846))); - assertTrue(map.get("verb").equals("GET")); + Assertions.assertThat(map).doesNotContainKey("Error"); + Instant ts = ZonedDateTime.of(2004, 3, 7, 16, 45, 56, 0, ZoneOffset.ofHours(-8)).toInstant(); + assertEquals(map.get("timestamp"), ts); + assertEquals(map.get("response"), 401); + assertEquals(map.get("ident"), Boolean.FALSE); + assertEquals(map.get("httpversion"), 1.1f); + assertEquals(map.get("bytes"), 12846L); + assertEquals("GET", map.get("verb")); } @Test - public void test002_httpd_access_colon() throws GrokException, IOException, ParseException { - Grok grok = Grok.create(ResourceManager.PATTERNS, - "%{IPORHOST:clientip} %{USER:ident:boolean} %{USER:auth} \\[%{HTTPDATE:timestamp:date:dd/MMM/yyyy:" - + "HH:mm:ss Z}\\] \"(?:%{WORD:verb:string} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion:float})?|" - + "%{DATA:rawrequest})\" %{NUMBER:response:int} (?:%{NUMBER:bytes:long}|-)"); + public void test002_httpd_access_colon() throws GrokException { + Grok grok = compiler.compile( + "%{IPORHOST:clientip} %{USER:ident:boolean} %{USER:auth} " + + "\\[%{HTTPDATE:timestamp:date:dd/MMM/yyyy:HH:mm:ss Z}\\] \"(?:%{WORD:verb:string} %{NOTSPACE:request}" + + "(?: HTTP/%{NUMBER:httpversion:float})?|%{DATA:rawrequest})\" %{NUMBER:response:int} " + + "(?:%{NUMBER:bytes:long}|-)"); - System.out.println(line); Match gm = grok.match(line); - gm.captures(); + Map map = gm.capture(); - assertNotEquals("{\"Error\":\"Error\"}", gm.toJson()); + Assertions.assertThat(map).doesNotContainKey("Error"); - Map map = gm.toMap(); - assertTrue(map.get("timestamp") - .equals(new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z").parse("07/Mar/2004:16:45:56 -0800"))); - assertTrue(map.get("response").equals(Integer.valueOf(401))); - assertTrue(map.get("ident").equals(Boolean.FALSE)); - assertTrue(map.get("httpversion").equals(Float.valueOf(1.1f))); - assertTrue(map.get("bytes").equals(Long.valueOf(12846))); - assertTrue(map.get("verb").equals("GET")); + Instant ts = ZonedDateTime.of(2004, 3, 7, 16, 45, 56, 0, ZoneOffset.ofHours(-8)).toInstant(); + assertEquals(map.get("timestamp"), ts); + assertEquals(map.get("response"), 401); + assertEquals(map.get("ident"), Boolean.FALSE); + assertEquals(map.get("httpversion"), 1.1f); + assertEquals(map.get("bytes"), 12846L); + assertEquals("GET", map.get("verb")); } - } diff --git a/src/test/java/io/krakens/grok/api/ApacheTest.java b/src/test/java/io/krakens/grok/api/ApacheTest.java index 2b920b4..a72e962 100644 --- a/src/test/java/io/krakens/grok/api/ApacheTest.java +++ b/src/test/java/io/krakens/grok/api/ApacheTest.java @@ -1,46 +1,53 @@ package io.krakens.grok.api; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.util.Map; import io.krakens.grok.api.exception.GrokException; +import com.google.common.io.Resources; +import org.assertj.core.api.Assertions; +import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; - @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class ApacheTest { public static final String LOG_FILE = "src/test/resources/access_log"; public static final String LOG_DIR_NASA = "src/test/resources/nasa/"; + private GrokCompiler compiler; + + @Before + public void setup() throws Exception { + compiler = GrokCompiler.newInstance(); + compiler.register(Resources.getResource(ResourceManager.PATTERNS).openStream()); + } + @Test public void test001_httpd_access() throws GrokException, IOException { - Grok grok = Grok.create(ResourceManager.PATTERNS, "%{COMMONAPACHELOG}"); + Grok grok = compiler.compile("%{COMMONAPACHELOG}"); BufferedReader br = new BufferedReader(new FileReader(LOG_FILE)); String line; System.out.println("Starting test with httpd log"); while ((line = br.readLine()) != null) { - //System.out.println(line); Match gm = grok.match(line); - gm.captures(); - assertNotNull(gm.toJson()); - assertNotEquals("{\"Error\":\"Error\"}", gm.toJson()); + final Map capture = gm.capture(); + Assertions.assertThat(capture).doesNotContainKey("Error"); } br.close(); } @Test public void test002_nasa_httpd_access() throws GrokException, IOException { - Grok grok = Grok.create(ResourceManager.PATTERNS, "%{COMMONAPACHELOG}"); + Grok grok = compiler.compile("%{COMMONAPACHELOG}"); System.out.println("Starting test with nasa log -- may take a while"); BufferedReader br; String line; @@ -48,11 +55,9 @@ public void test002_nasa_httpd_access() throws GrokException, IOException { for (File child : dir.listFiles()) { br = new BufferedReader(new FileReader(LOG_DIR_NASA + child.getName())); while ((line = br.readLine()) != null) { - //System.out.println(child.getName() + " " +line); Match gm = grok.match(line); - gm.captures(); - assertNotNull(gm.toJson()); - assertNotEquals("{\"Error\":\"Error\"}", gm.toJson()); + final Map capture = gm.capture(); + Assertions.assertThat(capture).doesNotContainKey("Error"); } br.close(); } diff --git a/src/test/java/io/krakens/grok/api/BasicTest.java b/src/test/java/io/krakens/grok/api/BasicTest.java index 0b44b54..27fe936 100644 --- a/src/test/java/io/krakens/grok/api/BasicTest.java +++ b/src/test/java/io/krakens/grok/api/BasicTest.java @@ -1,10 +1,11 @@ package io.krakens.grok.api; +import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import java.io.BufferedWriter; import java.io.File; +import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; @@ -13,20 +14,26 @@ import io.krakens.grok.api.exception.GrokException; +import com.google.common.io.Resources; +import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; - @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class BasicTest { - @Test - public void test001_compileFailOnInvalidExpression() throws GrokException { + private GrokCompiler compiler; - Grok grok = Grok.create(ResourceManager.PATTERNS, null); + @Before + public void setup() throws Exception { + compiler = GrokCompiler.newInstance(); + compiler.register(Resources.getResource(ResourceManager.PATTERNS).openStream()); + } - List badRegxp = new ArrayList(); + @Test + public void test001_compileFailOnInvalidExpression() throws GrokException { + List badRegxp = new ArrayList<>(); badRegxp.add("["); badRegxp.add("[foo"); badRegxp.add("?"); @@ -35,10 +42,10 @@ public void test001_compileFailOnInvalidExpression() throws GrokException { boolean thrown = false; - /** This should always throw. */ + /** This should always throw */ for (String regx : badRegxp) { try { - grok.compile(regx); + compiler.compile(regx); } catch (PatternSyntaxException e) { thrown = true; } @@ -49,48 +56,41 @@ public void test001_compileFailOnInvalidExpression() throws GrokException { @Test public void test002_compileSuccessValidExpression() throws GrokException { - - Grok grok = Grok.create(ResourceManager.PATTERNS); - - List regxp = new ArrayList(); + List regxp = new ArrayList<>(); regxp.add("[hello]"); regxp.add("(test)"); regxp.add("(?:hello)"); regxp.add("(?=testing)"); for (String regx : regxp) { - grok.compile(regx); + compiler.compile(regx); } } @Test public void test003_samePattern() throws GrokException { - Grok grok = Grok.create(ResourceManager.PATTERNS); - String pattern = "Hello World"; - grok.compile(pattern); + Grok grok = compiler.compile(pattern); assertEquals(pattern, grok.getOriginalGrokPattern()); } @Test public void test004_sameExpantedPatern() throws GrokException { - Grok grok = Grok.create(ResourceManager.PATTERNS); - - grok.addPattern("test", "hello world"); - grok.compile("%{test}"); + compiler.register("test", "hello world"); + Grok grok = compiler.compile("%{test}"); assertEquals("(?hello world)", grok.getNamedRegex()); } @Test public void test005_testLoadPatternFromFile() throws IOException, GrokException { File temp = File.createTempFile("grok-tmp-pattern", ".tmp"); - getClass(); BufferedWriter bw = new BufferedWriter(new FileWriter(temp)); bw.write("TEST \\d+"); bw.close(); - Grok grok = Grok.create(temp.getAbsolutePath()); - grok.compile("%{TEST}"); + GrokCompiler compiler = GrokCompiler.newInstance(); + compiler.register(new FileInputStream(temp)); + Grok grok = compiler.compile("%{TEST}"); assertEquals("(?\\d+)", grok.getNamedRegex()); temp.delete(); } diff --git a/src/test/java/io/krakens/grok/api/CaptureTest.java b/src/test/java/io/krakens/grok/api/CaptureTest.java index 4d5e93f..f870266 100644 --- a/src/test/java/io/krakens/grok/api/CaptureTest.java +++ b/src/test/java/io/krakens/grok/api/CaptureTest.java @@ -8,10 +8,12 @@ import static org.junit.Assert.fail; import java.util.List; +import java.util.Map; import io.krakens.grok.api.exception.GrokException; -import org.junit.BeforeClass; +import com.google.common.io.Resources; +import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; @@ -19,65 +21,65 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class CaptureTest { - static Grok grok; + GrokCompiler compiler; - @BeforeClass - public static void setUp() throws GrokException { - - grok = Grok.create(ResourceManager.PATTERNS, null); + @Before + public void setUp() throws Exception { + compiler = GrokCompiler.newInstance(); + compiler.register(Resources.getResource(ResourceManager.PATTERNS).openStream()); } @Test - public void test001_captureMathod() throws GrokException { - grok.addPattern("foo", ".*"); - grok.compile("%{foo}"); + public void test001_captureMathod() { + compiler.register("foo", ".*"); + Grok grok = compiler.compile("%{foo}"); Match match = grok.match("Hello World"); assertEquals("(?.*)", grok.getNamedRegex()); assertEquals("Hello World", match.getSubject()); - match.captures(); - assertEquals(1, match.toMap().size()); - assertEquals("Hello World", match.toMap().get("foo")); - assertEquals("{foo=Hello World}", match.toMap().toString()); + Map map = match.capture(); + assertEquals(1, map.size()); + assertEquals("Hello World", map.get("foo")); + assertEquals("{foo=Hello World}", map.toString()); } @Test - public void test002_captureMethodMulti() throws GrokException { - grok.addPattern("foo", ".*"); - grok.addPattern("bar", ".*"); - grok.compile("%{foo} %{bar}"); + public void test002_captureMathodMulti() throws GrokException { + compiler.register("foo", ".*"); + compiler.register("bar", ".*"); + Grok grok = compiler.compile("%{foo} %{bar}"); Match match = grok.match("Hello World"); assertEquals("(?.*) (?.*)", grok.getNamedRegex()); assertEquals("Hello World", match.getSubject()); - match.captures(); - assertEquals(2, match.toMap().size()); - assertEquals("Hello", match.toMap().get("foo")); - assertEquals("World", match.toMap().get("bar")); - assertEquals("{bar=World, foo=Hello}", match.toMap().toString()); + Map map = match.capture(); + assertEquals(2, map.size()); + assertEquals("Hello", map.get("foo")); + assertEquals("World", map.get("bar")); + assertEquals("{bar=World, foo=Hello}", map.toString()); } @Test public void test003_captureMathodNasted() throws GrokException { - grok.addPattern("foo", "\\w+ %{bar}"); - grok.addPattern("bar", "\\w+"); - grok.compile("%{foo}"); + compiler.register("foo", "\\w+ %{bar}"); + compiler.register("bar", "\\w+"); + Grok grok = compiler.compile("%{foo}"); Match match = grok.match("Hello World"); assertEquals("(?\\w+ (?\\w+))", grok.getNamedRegex()); assertEquals("Hello World", match.getSubject()); - match.captures(); - assertEquals(2, match.toMap().size()); - assertEquals("Hello World", match.toMap().get("foo")); - assertEquals("World", match.toMap().get("bar")); - assertEquals("{bar=World, foo=Hello World}", match.toMap().toString()); + Map map = match.capture(); + assertEquals(2, map.size()); + assertEquals("Hello World", map.get("foo")); + assertEquals("World", map.get("bar")); + assertEquals("{bar=World, foo=Hello World}", map.toString()); } @Test public void test004_captureNastedRecustion() throws GrokException { - grok.addPattern("foo", "%{foo}"); + compiler.register("foo", "%{foo}"); boolean thrown = false; /** Must raise `Deep recursion pattern` execption */ try { - grok.compile("%{foo}"); - } catch (GrokException e) { + compiler.compile("%{foo}"); + } catch (Exception e) { thrown = true; } assertTrue(thrown); @@ -87,62 +89,62 @@ public void test004_captureNastedRecustion() throws GrokException { public void test005_captureSubName() throws GrokException { String name = "foo"; String subname = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_abcdef"; - grok.addPattern(name, "\\w+"); - grok.compile("%{" + name + ":" + subname + "}"); + compiler.register(name, "\\w+"); + Grok grok = compiler.compile("%{" + name + ":" + subname + "}"); Match match = grok.match("Hello"); - match.captures(); - assertEquals(1, match.toMap().size()); - assertEquals("Hello", match.toMap().get(subname).toString()); + Map map = match.capture(); + assertEquals(1, map.size()); + assertEquals("Hello", map.get(subname).toString()); assertEquals("{abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_abcdef=Hello}", - match.toMap().toString()); + map.toString()); } @Test public void test006_captureOnlyNamed() throws GrokException { - grok.addPattern("abcdef", "[a-zA-Z]+"); - grok.addPattern("ghijk", "\\d+"); - grok.compile("%{abcdef:abcdef}%{ghijk}", true); + compiler.register("abcdef", "[a-zA-Z]+"); + compiler.register("ghijk", "\\d+"); + Grok grok = compiler.compile("%{abcdef:abcdef}%{ghijk}", true); Match match = grok.match("abcdef12345"); - match.captures(); - assertEquals(match.toMap().size(), 1); - assertNull(match.toMap().get("ghijk")); - assertEquals(match.toMap().get("abcdef"), "abcdef"); + Map map = match.capture(); + assertEquals(map.size(), 1); + assertNull(map.get("ghijk")); + assertEquals(map.get("abcdef"), "abcdef"); } @SuppressWarnings("unchecked") @Test public void test007_captureDuplicateName() throws GrokException { - grok.compile("%{INT:id} %{INT:id}"); + Grok grok = compiler.compile("%{INT:id} %{INT:id}"); Match match = grok.match("123 456"); - match.captures(); - assertEquals(match.toMap().size(), 1); - assertEquals(((List) (match.toMap().get("id"))).size(), 2); - assertEquals(((List) (match.toMap().get("id"))).get(0), "123"); - assertEquals(((List) (match.toMap().get("id"))).get(1), "456"); + Map map = match.capture(); + assertEquals(map.size(), 1); + assertEquals(((List) (map.get("id"))).size(), 2); + assertEquals(((List) (map.get("id"))).get(0), "123"); + assertEquals(((List) (map.get("id"))).get(1), "456"); } @SuppressWarnings("unchecked") @Test public void test008_flattenDuplicateKeys() throws GrokException { - grok.compile("(?:foo %{INT:id} bar|bar %{INT:id} foo)"); + Grok grok = compiler.compile("(?:foo %{INT:id} bar|bar %{INT:id} foo)"); Match match = grok.match("foo 123 bar"); - match.capturesFlattened(); - assertEquals(match.toMap().size(), 1); - assertEquals(match.toMap().get("id"), "123"); - Match match2 = grok.match("bar 123 foo"); - match2.capturesFlattened(); - assertEquals(match2.toMap().size(), 1); - assertEquals(match2.toMap().get("id"), "123"); + Map map = match.captureFlattened(); + assertEquals(map.size(), 1); + assertEquals(map.get("id"), "123"); + Match m2 = grok.match("bar 123 foo"); + map = m2.captureFlattened(); + assertEquals(map.size(), 1); + assertEquals(map.get("id"), "123"); - grok.compile("%{INT:id} %{INT:id}"); - Match match1 = grok.match("123 456"); + grok = compiler.compile("%{INT:id} %{INT:id}"); + Match m3 = grok.match("123 456"); try { - match1.capturesFlattened(); + m3.captureFlattened(); fail("should report error due tu ambiguity"); } catch (RuntimeException e) { - assertThat(e.getMessage(), containsString("has multiple non-null values, this is not allowed in flattened mode")); + assertThat(e.getMessage(), + containsString("has multiple non-null values, this is not allowed in flattened mode")); } } - } diff --git a/src/test/java/io/krakens/grok/api/ConcurrencyTest.java b/src/test/java/io/krakens/grok/api/ConcurrencyTest.java deleted file mode 100644 index d48d47b..0000000 --- a/src/test/java/io/krakens/grok/api/ConcurrencyTest.java +++ /dev/null @@ -1,57 +0,0 @@ -package io.krakens.grok.api; - -import static org.hamcrest.core.IsEqual.equalTo; -import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.assertEquals; - -import java.util.Collections; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.junit.Assert; -import org.junit.Test; - -public class ConcurrencyTest { - - /** - * We will test this by setting up two threads. - * asserting on the hash values the instances generate for each thread - */ - @Test - public void test_001_concurrent_match() throws InterruptedException, ExecutionException { - - // Setup callable method that reports the hashcode for the thread - int threadCount = 2; - Callable task = new Callable() { - @Override - public Integer call() { - return Match.getInstance().hashCode(); - } - }; - - // Create n tasks to execute - List> tasks = Collections.nCopies(threadCount, task); - - // Execute the task for both tasks - ExecutorService es = Executors.newFixedThreadPool(threadCount); - List> futures = es.invokeAll(tasks); - int hash1 = futures.get(0).get(); - int hash2 = futures.get(1).get(); - - // The two hashcodes must NOT be equal - Assert.assertThat(hash1, not(equalTo(hash2))); - } - - @Test - public void test_002_match_within_instance() { - // Verify that the instances are equal for the same thread - Match m1 = Match.getInstance(); - Match m2 = Match.getInstance(); - assertEquals(m1, m2); - } - -} diff --git a/src/test/java/io/krakens/grok/api/GrokDocumentationTest.java b/src/test/java/io/krakens/grok/api/GrokDocumentationTest.java new file mode 100644 index 0000000..62de9fb --- /dev/null +++ b/src/test/java/io/krakens/grok/api/GrokDocumentationTest.java @@ -0,0 +1,31 @@ +package io.krakens.grok.api; + +import java.util.Map; + +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public class GrokDocumentationTest { + + @Test + public void assureCodeInReadmeWorks() { + /* Create a new grokCompiler instance */ + GrokCompiler grokCompiler = GrokCompiler.newInstance(); + grokCompiler.registerDefaultPatterns(); + + /* Grok pattern to compile, here httpd logs */ + final Grok grok = grokCompiler.compile("%{COMBINEDAPACHELOG}"); + + /* Line of log to match */ + String log = "112.169.19.192 - - [06/Mar/2013:01:36:30 +0900] \"GET / HTTP/1.1\" 200 44346 \"-\" " + + "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.22 (KHTML, like Gecko) " + + "Chrome/25.0.1364.152 Safari/537.22\""; + + Match gm = grok.match(log); + + /* Get the map with matches */ + final Map capture = gm.capture(); + + Assertions.assertThat(capture).hasSize(22); + } +} diff --git a/src/test/java/io/krakens/grok/api/GrokListTest.java b/src/test/java/io/krakens/grok/api/GrokListTest.java index e0e95fb..8caff86 100644 --- a/src/test/java/io/krakens/grok/api/GrokListTest.java +++ b/src/test/java/io/krakens/grok/api/GrokListTest.java @@ -5,9 +5,12 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import io.krakens.grok.api.exception.GrokException; +import com.google.common.io.Resources; +import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; @@ -15,9 +18,17 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class GrokListTest { + GrokCompiler compiler; + + @Before + public void setUp() throws Exception { + compiler = GrokCompiler.newInstance(); + compiler.register(Resources.getResource(ResourceManager.PATTERNS).openStream()); + } + @Test public void test_001() throws GrokException { - List logs = new ArrayList(); + List logs = new ArrayList<>(); logs.add("178.21.82.201"); logs.add("11.178.94.216"); @@ -29,22 +40,20 @@ public void test_001() throws GrokException { logs.add("170.36.40.12"); logs.add("124.2.84.36"); - Grok grok = Grok.create(ResourceManager.PATTERNS, "%{IP}"); - List json = grok.captures(logs); - assertNotNull(json); + Grok grok = compiler.compile("%{IP}"); + ArrayList> capture = grok.capture(logs); + assertNotNull(capture); int counter = 0; - for (String elem : json) { + for (Map elem : capture) { assertNotNull(elem); assertEquals(elem, grok.capture(logs.get(counter))); counter++; - //assert } - } @Test public void test_002() throws GrokException { - List logs = new ArrayList(); + List logs = new ArrayList<>(); logs.add("178.21.82.201"); logs.add("11.178.94.216"); @@ -56,17 +65,14 @@ public void test_002() throws GrokException { logs.add("170.36.40.12"); logs.add("124.2.84.36"); - Grok grok = Grok.create(ResourceManager.PATTERNS, "%{IP}"); - List json = grok.captures(logs); - assertNotNull(json); + Grok grok = compiler.compile("%{IP}"); + ArrayList> capture = grok.capture(logs); + assertNotNull(capture); int counter = 0; - for (String elem : json) { - System.out.println(elem); + for (Map elem : capture) { assertNotNull(elem); assertEquals(elem, grok.capture(logs.get(counter))); counter++; - //assert } - } } diff --git a/src/test/java/io/krakens/grok/api/GrokTest.java b/src/test/java/io/krakens/grok/api/GrokTest.java index d764110..0053348 100644 --- a/src/test/java/io/krakens/grok/api/GrokTest.java +++ b/src/test/java/io/krakens/grok/api/GrokTest.java @@ -3,22 +3,31 @@ import static java.lang.String.format; import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.BufferedReader; import java.io.FileReader; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; +import java.util.Locale; +import java.util.Map; import io.krakens.grok.api.exception.GrokException; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Resources; +import org.assertj.core.api.Assertions; +import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; @@ -27,339 +36,253 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class GrokTest { - private Grok grok = Grok.EMPTY; + static { + Locale.setDefault(Locale.ROOT); + } + + GrokCompiler compiler; + + @Before + public void setUp() throws Exception { + compiler = GrokCompiler.newInstance(); + compiler.register(Resources.getResource(ResourceManager.PATTERNS).openStream()); + } @Test public void test000_basic() { - Grok grok = new Grok(); + GrokCompiler compiler = GrokCompiler.newInstance(); boolean thrown = false; - // expected exception try { - grok.addPatternFromFile("/good/luck"); - } catch (GrokException e) { - thrown = true; - } - assertTrue(thrown); - - thrown = false; - - try { - grok.addPattern(null, ""); - } catch (GrokException e) { + compiler.register(null, ""); + } catch (NullPointerException e) { thrown = true; } assertTrue(thrown); + } - thrown = false; - try { - grok.copyPatterns(null); - } catch (GrokException e) { - thrown = true; - } - assertTrue(thrown); + @Test(expected = Exception.class) + public void test_throwExceptionIfPatternIsNull() { + compiler.compile(null); + } - thrown = false; - try { - grok.copyPatterns(new HashMap()); - } catch (GrokException e) { - thrown = true; - } - assertTrue(thrown); + @Test(expected = Exception.class) + public void test_throwExceptionIfPatternIsEmptyString() { + compiler.compile(""); } - @Test - public void test000_dummy() throws Throwable { - grok.addPatternFromFile(ResourceManager.PATTERNS); - boolean thrown = false; - /** This check if grok throw */ - try { - grok.compile(null); - } catch (GrokException e) { - thrown = true; - } - assertTrue(thrown); - thrown = false; - try { - grok.compile(""); - } catch (GrokException e) { - thrown = true; - } - assertTrue(thrown); - thrown = false; - try { - grok.compile(" "); - } catch (GrokException e) { - thrown = true; - } - assertTrue(thrown); + @Test(expected = Exception.class) + public void test_throwExceptionIfPatternContainsOnlyBlanks() { + compiler.compile(" "); } @Test - public void test001_static_metod_factory() throws Throwable { + public void test001_static_metod_factory() { - Grok staticGrok = Grok.create(ResourceManager.PATTERNS, "%{USERNAME}"); + Grok staticGrok = compiler.compile("%{USERNAME}"); Match gm = staticGrok.match("root"); - gm.captures(); - assertEquals("{USERNAME=root}", gm.toMap().toString()); + Map map = gm.capture(); + assertEquals("{USERNAME=root}", map.toString()); gm = staticGrok.match("r00t"); - gm.captures(); - assertEquals("{USERNAME=r00t}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{USERNAME=r00t}", map.toString()); gm = staticGrok.match("guest"); - gm.captures(); - assertEquals("{USERNAME=guest}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{USERNAME=guest}", map.toString()); gm = staticGrok.match("guest1234"); - gm.captures(); - assertEquals("{USERNAME=guest1234}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{USERNAME=guest1234}", map.toString()); gm = staticGrok.match("john doe"); - gm.captures(); - assertEquals("{USERNAME=john}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{USERNAME=john}", map.toString()); } - @Test - public void test001_username() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{USERNAME}"); + public void test001_username2() { + Grok grok = compiler.compile("%{USER}"); Match gm = grok.match("root"); - gm.captures(); - assertEquals("{USERNAME=root}", gm.toMap().toString()); + Map map = gm.capture(); + assertEquals("{USER=root}", map.toString()); gm = grok.match("r00t"); - gm.captures(); - assertEquals("{USERNAME=r00t}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{USER=r00t}", map.toString()); gm = grok.match("guest"); - gm.captures(); - assertEquals("{USERNAME=guest}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{USER=guest}", map.toString()); gm = grok.match("guest1234"); - gm.captures(); - assertEquals("{USERNAME=guest1234}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{USER=guest1234}", map.toString()); gm = grok.match("john doe"); - gm.captures(); - assertEquals("{USERNAME=john}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{USER=john}", map.toString()); } @Test - public void test001_username2() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{USER}"); - - Match gm = grok.match("root"); - gm.captures(); - assertEquals("{USER=root}", gm.toMap().toString()); - - gm = grok.match("r00t"); - gm.captures(); - assertEquals("{USER=r00t}", gm.toMap().toString()); - - gm = grok.match("guest"); - gm.captures(); - assertEquals("{USER=guest}", gm.toMap().toString()); - - gm = grok.match("guest1234"); - gm.captures(); - assertEquals("{USER=guest1234}", gm.toMap().toString()); - - gm = grok.match("john doe"); - gm.captures(); - assertEquals("{USER=john}", gm.toMap().toString()); - } - - @Test - public void test002_numbers() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{NUMBER}"); + public void test002_numbers() { + Grok grok = compiler.compile("%{NUMBER}"); Match gm = grok.match("-42"); - gm.captures(); - assertEquals("{NUMBER=-42}", gm.toMap().toString()); + Map map = gm.capture(); + assertEquals("{NUMBER=-42}", map.toString()); } @Test - public void test003_word() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{WORD}"); + public void test003_word() { + Grok grok = compiler.compile("%{WORD}"); Match gm = grok.match("a"); - gm.captures(); - assertEquals("{WORD=a}", gm.toMap().toString()); + Map map = gm.capture(); + assertEquals("{WORD=a}", map.toString()); gm = grok.match("abc"); - gm.captures(); - assertEquals("{WORD=abc}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{WORD=abc}", map.toString()); } @Test - public void test004_space() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{SPACE}"); + public void test004_space() { + Grok grok = compiler.compile("%{SPACE}"); Match gm = grok.match("abc dc"); - gm.captures(); - assertEquals("{SPACE=}", gm.toMap().toString()); + Map map = gm.capture(); + assertEquals("{SPACE=}", map.toString()); } @Test - public void test004_number() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{NUMBER}"); + public void test004_number() { + Grok grok = compiler.compile("%{NUMBER}"); Match gm = grok.match("Something costs $55.4!"); - gm.captures(); - assertEquals("{NUMBER=55.4}", gm.toMap().toString()); + Map map = gm.capture(); + assertEquals("{NUMBER=55.4}", map.toString()); } @Test - public void test005_notSpace() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{NOTSPACE}"); + public void test005_notSpace() { + Grok grok = compiler.compile("%{NOTSPACE}"); Match gm = grok.match("abc dc"); - gm.captures(); - assertEquals("{NOTSPACE=abc}", gm.toMap().toString()); + Map map = gm.capture(); + assertEquals("{NOTSPACE=abc}", map.toString()); } @Test - public void test006_quotedString() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{QUOTEDSTRING:text}"); + public void test006_quotedString() { + Grok grok = compiler.compile("%{QUOTEDSTRING:text}"); Match gm = grok.match("\"abc dc\""); - gm.captures(); - assertEquals("{text=abc dc}", gm.toMap().toString()); - + Map map = gm.capture(); + assertEquals("{text=abc dc}", map.toString()); } @Test - public void test007_uuid() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{UUID}"); + public void test007_uuid() { + Grok grok = compiler.compile("%{UUID}"); Match gm = grok.match("61243740-4786-11e3-86a7-0002a5d5c51b"); - gm.captures(); - assertEquals("{UUID=61243740-4786-11e3-86a7-0002a5d5c51b}", gm.toMap().toString()); + Map map = gm.capture(); + assertEquals("{UUID=61243740-4786-11e3-86a7-0002a5d5c51b}", map.toString()); gm = grok.match("7F8C7CB0-4786-11E3-8F96-0800200C9A66"); - gm.captures(); - assertEquals("{UUID=7F8C7CB0-4786-11E3-8F96-0800200C9A66}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{UUID=7F8C7CB0-4786-11E3-8F96-0800200C9A66}", map.toString()); gm = grok.match("03A8413C-F604-4D21-8F4D-24B19D98B5A7"); - gm.captures(); - assertEquals("{UUID=03A8413C-F604-4D21-8F4D-24B19D98B5A7}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{UUID=03A8413C-F604-4D21-8F4D-24B19D98B5A7}", map.toString()); } @Test - public void test008_mac() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{MAC}"); + public void test008_mac() { + Grok grok = compiler.compile("%{MAC}"); Match gm = grok.match("5E:FF:56:A2:AF:15"); - gm.captures(); - assertEquals("{MAC=5E:FF:56:A2:AF:15}", gm.toMap().toString()); + Map map = gm.capture(); + assertEquals("{MAC=5E:FF:56:A2:AF:15}", map.toString()); } @Test - public void test009_ipOrPort() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{IPORHOST}"); + public void test009_ipOrPort() { + Grok grok = compiler.compile("%{IPORHOST}"); Match gm = grok.match("www.google.fr"); - gm.captures(); - assertEquals("{IPORHOST=www.google.fr}", gm.toMap().toString()); + Map map = gm.capture(); + assertEquals("{IPORHOST=www.google.fr}", map.toString()); gm = grok.match("www.google.com"); - gm.captures(); - assertEquals("{IPORHOST=www.google.com}", gm.toMap().toString()); + map = gm.capture(); + assertEquals("{IPORHOST=www.google.com}", map.toString()); } @Test - public void test010_hostPort() throws Throwable { - - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{HOSTPORT}"); + public void test010_hostPort() { + Grok grok = compiler.compile("%{HOSTPORT}"); Match gm = grok.match("www.google.fr:80"); - gm.captures(); - assertEquals("{HOSTPORT=www.google.fr:80, IPORHOST=www.google.fr, PORT=80}", gm.toMap() - .toString()); + Map map = gm.capture(); + assertEquals(ImmutableMap.of( + "HOSTPORT", "www.google.fr:80", + "IPORHOST", "www.google.fr", + "PORT", "80"), map); } @Test - public void test011_combiledApacheLog() throws Throwable { + public void test011_combineApache() { + Grok grok = compiler.compile("%{COMBINEDAPACHELOG}"); - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{COMBINEDAPACHELOG}"); - - Match gm = grok - .match("112.169.19.192 - - [06/Mar/2013:01:36:30 +0900] \"GET / HTTP/1.1\" 200 44346 \"-\" \"Mozilla/5.0 " - + "(Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.22 (KHTML, like Gecko) " + Match gm = + grok.match("112.169.19.192 - - [06/Mar/2013:01:36:30 +0900] \"GET / HTTP/1.1\" 200 44346 \"-\" " + + "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.22 (KHTML, like Gecko) " + "Chrome/25.0.1364.152 Safari/537.22\""); - gm.captures(); - assertNotNull(gm.toJson()); + Map map = gm.capture(); assertEquals( - gm.toMap().get("agent").toString(), + map.get("agent").toString(), "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.22 (KHTML, like Gecko) " + "Chrome/25.0.1364.152 Safari/537.22"); - assertEquals(gm.toMap().get("clientip").toString(), "112.169.19.192"); - assertEquals(gm.toMap().get("httpversion").toString(), "1.1"); - assertEquals(gm.toMap().get("timestamp").toString(), "06/Mar/2013:01:36:30 +0900"); - assertEquals(gm.toMap().get("TIME").toString(), "01:36:30"); + assertEquals(map.get("clientip").toString(), "112.169.19.192"); + assertEquals(map.get("httpversion").toString(), "1.1"); + assertEquals(map.get("timestamp").toString(), "06/Mar/2013:01:36:30 +0900"); + assertEquals(map.get("TIME").toString(), "01:36:30"); gm = - grok.match( - "112.169.19.192 - - [06/Mar/2013:01:36:30 +0900] \"GET /wp-content/plugins/easy-table/themes/default/" - + "style.css?ver=1.0 HTTP/1.1\" 304 - \"http://www.nflabs.com/\" \"Mozilla/5.0 (Macintosh; Intel " - + "Mac OS X 10_8_2) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.152 Safari/537.22\""); - gm.captures(); - assertNotNull(gm.toJson()); - // System.out.println(gm.toJson()); + grok.match("112.169.19.192 - - [06/Mar/2013:01:36:30 +0900] \"GET " + + "/wp-content/plugins/easy-table/themes/default/style.css?ver=1.0 HTTP/1.1\" " + + "304 - \"http://www.nflabs.com/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) " + + "AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.152 Safari/537.22\""); + map = gm.capture(); assertEquals( - gm.toMap().get("agent").toString(), + map.get("agent").toString(), "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.22 (KHTML, like Gecko) " + "Chrome/25.0.1364.152 Safari/537.22"); - assertEquals(gm.toMap().get("clientip").toString(), "112.169.19.192"); - assertEquals(gm.toMap().get("httpversion").toString(), "1.1"); - assertEquals(gm.toMap().get("request").toString(), + assertEquals(map.get("clientip").toString(), "112.169.19.192"); + assertEquals(map.get("httpversion").toString(), "1.1"); + assertEquals(map.get("request").toString(), "/wp-content/plugins/easy-table/themes/default/style.css?ver=1.0"); - assertEquals(gm.toMap().get("TIME").toString(), "01:36:30"); - - // assertEquals("{HOSTPORT=www.google.fr:80, IPORHOST=www.google.fr, PORT=80}", - // gm.toMap().toString()); - + assertEquals(map.get("TIME").toString(), "01:36:30"); } @Test - public void test012_day() throws Throwable { + public void test012_day() { - Grok grok = Grok.create(ResourceManager.PATTERNS, "%{DAY}"); + Grok grok = compiler.compile("%{DAY}"); - List days = new ArrayList(); + List days = new ArrayList<>(); days.add("Mon"); days.add("Monday"); days.add("Tue"); @@ -378,84 +301,81 @@ public void test012_day() throws Throwable { int counter = 0; for (String day : days) { Match match = grok.match(day); - match.captures(); - assertNotNull(match.toMap()); - assertEquals(match.toMap().get("DAY"), days.get(counter)); + Map map = match.capture(); + assertNotNull(map); + assertEquals(map.get("DAY"), days.get(counter)); counter++; } } @Test public void test013_IpSet() throws Throwable { - Grok grok = Grok.create(ResourceManager.PATTERNS, "%{IP}"); + Grok grok = compiler.compile("%{IP}"); - BufferedReader br = new BufferedReader(new FileReader(ResourceManager.IP)); + BufferedReader br = new BufferedReader(new FileReader(Resources.getResource(ResourceManager.IP).getFile())); String line; System.out.println("Starting test with ip"); while ((line = br.readLine()) != null) { Match gm = grok.match(line); - gm.captures(); - assertNotNull(gm.toJson()); - assertNotEquals("{\"Error\":\"Error\"}", gm.toJson()); - assertEquals(gm.toMap().get("IP"), line); + final Map map = gm.capture(); + Assertions.assertThat(map).doesNotContainKey("Error"); + assertEquals(map.get("IP"), line); } } @Test - public void test014_month() throws Throwable { + public void test014_month() { - Grok grok = Grok.create(ResourceManager.PATTERNS, "%{MONTH}"); + Grok grok = compiler.compile("%{MONTH}"); - String[] array = {"Jan", "January", "Feb", "February", "Mar", "March", "Apr", "April", "May", "Jun", "June", + String[] months = {"Jan", "January", "Feb", "February", "Mar", "March", "Apr", "April", "May", "Jun", "June", "Jul", "July", "Aug", "August", "Sep", "September", "Oct", "October", "Nov", "November", "Dec", "December"}; - List months = new ArrayList(Arrays.asList(array)); int counter = 0; for (String month : months) { Match match = grok.match(month); - match.captures(); - assertNotNull(match.toMap()); - assertEquals(match.toMap().get("MONTH"), months.get(counter)); + Map map = match.capture(); + assertNotNull(map); + assertEquals(map.get("MONTH"), months[counter]); counter++; } } @Test public void test015_iso8601() throws GrokException { - Grok grok = Grok.create(ResourceManager.PATTERNS, "%{TIMESTAMP_ISO8601}"); - - String[] array = - {"2001-01-01T00:00:00", - "1974-03-02T04:09:09", - "2010-05-03T08:18:18+00:00", - "2004-07-04T12:27:27-00:00", - "2001-09-05T16:36:36+0000", - "2001-11-06T20:45:45-0000", - "2001-12-07T23:54:54Z", - "2001-01-01T00:00:00.123456", - "1974-03-02T04:09:09.123456", - "2010-05-03T08:18:18.123456+00:00", - "2004-07-04T12:27:27.123456-00:00", - "2001-09-05T16:36:36.123456+0000", - "2001-11-06T20:45:45.123456-0000", - "2001-12-07T23:54:54.123456Z"}; - - List times = new ArrayList(Arrays.asList(array)); + Grok grok = compiler.compile("%{TIMESTAMP_ISO8601}"); + + String[] times = { + "2001-01-01T00:00:00", + "1974-03-02T04:09:09", + "2010-05-03T08:18:18+00:00", + "2004-07-04T12:27:27-00:00", + "2001-09-05T16:36:36+0000", + "2001-11-06T20:45:45-0000", + "2001-12-07T23:54:54Z", + "2001-01-01T00:00:00.123456", + "1974-03-02T04:09:09.123456", + "2010-05-03T08:18:18.123456+00:00", + "2004-07-04T12:27:27.123456-00:00", + "2001-09-05T16:36:36.123456+0000", + "2001-11-06T20:45:45.123456-0000", + "2001-12-07T23:54:54.123456Z"}; + int counter = 0; for (String time : times) { Match match = grok.match(time); - match.captures(); - assertNotNull(match.toMap()); - assertEquals(match.toMap().get("TIMESTAMP_ISO8601"), times.get(counter)); + Map map = match.capture(); + assertNotNull(map); + assertEquals(map.get("TIMESTAMP_ISO8601"), times[counter]); counter++; } } @Test public void test016_uri() throws GrokException { - Grok grok = Grok.create(ResourceManager.PATTERNS, "%{URI}"); + Grok grok = compiler.compile("%{URI}"); - String[] array = { + String[] uris = { "http://www.google.com", "telnet://helloworld", "http://www.example.com/", @@ -480,40 +400,39 @@ public void test016_uri() throws GrokException { "foo://user@somehost.com:12345/foo.bar/baz/fizz?test=1&sink&foo=4", "http://www.google.com/search?hl=en&source=hp&q=hello+world+%5E%40%23%24&btnG=Google+Search", "http://www.freebsd.org/cgi/url.cgi?ports/sysutils/grok/pkg-descr", - "http://www.google.com/search?q=CAPTCHA+ssh&start=0&ie=utf-8&oe=utf-8&client=firefox-a&rls=org.mozilla:en-US:" - + "official", + "http://www.google.com/search?q=CAPTCHA+ssh&start=0&ie=utf-8&oe=utf-8&client=firefox-a" + + "&rls=org.mozilla:en-US:official", "svn+ssh://somehost:12345/testing"}; - List uris = new ArrayList(Arrays.asList(array)); int counter = 0; for (String uri : uris) { Match match = grok.match(uri); - match.captures(); - assertNotNull(match.toMap()); - assertEquals(match.toMap().get("URI"), uris.get(counter)); - assertNotNull(match.toMap().get("URIPROTO")); + Map map = match.capture(); + assertNotNull(map); + assertEquals(map.get("URI"), uris[counter]); + assertNotNull(map.get("URIPROTO")); counter++; } } @Test public void test017_nonMachingList() throws GrokException { - Grok grok = Grok.create(ResourceManager.PATTERNS, "%{URI}"); + Grok grok = compiler.compile("%{URI}"); - String[] array = { + String[] uris = { "http://www.google.com", "telnet://helloworld", "", - "svn+ssh://somehost:12345/testing"}; + "svn+ssh://somehost:12345/testing" + }; - List uris = new ArrayList(Arrays.asList(array)); int counter = 0; for (String uri : uris) { Match match = grok.match(uri); - match.captures(); - assertNotNull(match.toMap()); + Map map = match.capture(); + assertNotNull(map); if (counter == 2) { - assertEquals(Collections.EMPTY_MAP, match.toMap()); + assertEquals(Collections.EMPTY_MAP, map); } counter++; } @@ -522,19 +441,17 @@ public void test017_nonMachingList() throws GrokException { @Test public void test018_namedOnlySimpleCase() throws GrokException { - Grok grok = Grok.create(ResourceManager.PATTERNS); + compiler.register("WORD", "foo|bar"); + compiler.register("TEXT", "<< %{WORD}+ >>"); - grok.addPattern("WORD", "foo|bar"); - grok.addPattern("TEXT", "<< %{WORD}+ >>"); - - grok.compile("%{TEXT:text}", true); + Grok grok = compiler.compile("%{TEXT:text}", true); String text = "<< barfoobarfoo >>"; Match match = grok.match(text); - match.captures(); + Map map = match.capture(); assertEquals("unable to parse: " + text, text, - match.toMap().get("text")); + map.get("text")); } @Test @@ -548,14 +465,11 @@ public void test019_namedOnlyAllCases() throws GrokException { private void testPatternRepetitions(boolean namedOnly, String pattern) throws GrokException { String description = format("[readonly:%s pattern:%s] ", namedOnly, pattern); - ; - - Grok grok = Grok.create(ResourceManager.PATTERNS); - grok.addPattern("WORD", pattern); - grok.addPattern("TEXT", "<< %{WORD}+ >>"); + compiler.register("WORD", pattern); + compiler.register("TEXT", "<< %{WORD}+ >>"); - grok.compile("%{TEXT:text}", namedOnly); + Grok grok = compiler.compile("%{TEXT:text}", namedOnly); assertMatches(description, grok, "<< foo >>"); assertMatches(description, grok, "<< foobar >>"); assertMatches(description, grok, "<< foofoobarbar >>"); @@ -564,93 +478,143 @@ private void testPatternRepetitions(boolean namedOnly, String pattern) throws Gr private void assertMatches(String description, Grok grok, String text) { Match match = grok.match(text); - match.captures(); + Map map = match.capture(); assertEquals(format("%s: unable to parse '%s'", description, text), text, - match.toMap().get("text")); - } - - /* see: https://github.com/thekrakken/java-grok/issues/64 */ - @Test - public void testDisablingAutomaticConversion() throws GrokException { - String input = "client id: \"foo\" \"bar\""; - String pattern = "(?client id): (?.*)"; - - Grok grok = new Grok(); - grok.disableAutomaticConversion(); - grok.compile(pattern, false); - Match gm = grok.match(input); - gm.captures(); - assertEquals("\"foo\" \"bar\"", gm.toMap().get("clientid")); + map.get("text")); } @Test public void test020_postfix_patterns() throws Throwable { - final Grok grok = Grok.create("src/main/resources/patterns/postfix"); - grok.addPatternFromFile("src/main/resources/patterns/patterns"); - grok.compile("%{POSTFIX_SMTPD}", false); + GrokCompiler compiler = GrokCompiler.newInstance(); + compiler.register(Resources.getResource("patterns/postfix").openStream()); + compiler.register(Resources.getResource("patterns/patterns").openStream()); + Grok grok = compiler.compile("%{POSTFIX_SMTPD}", false); assertTrue(grok.getPatterns().containsKey("POSTFIX_SMTPD")); } @Test public void test021_postfix_patterns_with_named_captures_only() throws Throwable { - final Grok grok = Grok.create("src/main/resources/patterns/postfix"); - grok.addPatternFromFile("src/main/resources/patterns/patterns"); - grok.compile("%{POSTFIX_SMTPD}", true); + GrokCompiler compiler = GrokCompiler.newInstance(); + compiler.register(Resources.getResource("patterns/postfix").openStream()); + compiler.register(Resources.getResource("patterns/patterns").openStream()); + Grok grok = compiler.compile("%{POSTFIX_SMTPD}", true); assertTrue(grok.getPatterns().containsKey("POSTFIX_SMTPD")); } @Test - public void test022_named_captures_with_missing_definition() throws Throwable { + public void test022_named_captures_with_missing_definition() { ensureAbortsWithDefinitionMissing("FOO %{BAR}", "%{FOO}", true); } @Test - public void test023_captures_with_missing_definition() throws Throwable { + public void test023_captures_with_missing_definition() { ensureAbortsWithDefinitionMissing("FOO %{BAR}", "%{FOO:name}", false); } @Test - public void test024_captures_with_missing_definition() throws Throwable { + public void test024_captures_with_missing_definition() { ensureAbortsWithDefinitionMissing("FOO %{BAR}", "%{FOO}", false); } @Test - public void allowClassPathPatternFiles() throws GrokException { - final Grok grok = new Grok(); - grok.addPatternFromClasspath("/patterns/patterns"); - grok.compile("%{USERNAME}", false); + public void test025_datetime_pattern_with_slashes() throws Throwable { + final ZonedDateTime expectedDate = ZonedDateTime.of(2015, 7, 31, 0, 0, 0, 0, ZoneOffset.UTC); + + final Grok grok = compiler.compile("Foo %{DATA:result;date;yyyy/MM/dd} Bar"); + + final Match gm = grok.match("Foo 2015/07/31 Bar"); + + assertEquals(1, gm.getMatch().groupCount()); + assertEquals(expectedDate.toInstant(), gm.capture().get("result")); } @Test - public void test025_keep_empty_captures() throws Throwable { - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{POSINT:pos}|%{INT:int}"); - Match gm = grok.match("-42"); - grok.setKeepEmptyCaptures(false); - gm.captures(); - assertEquals("{int=-42}", gm.toMap().toString()); - grok.setKeepEmptyCaptures(true); - gm.captures(); - assertEquals("{int=-42, pos=null}", gm.toMap().toString()); + public void test026_datetime_pattern_with_with_dots() throws Throwable { + final ZonedDateTime expectedDate = ZonedDateTime.of(2015, 7, 31, 0, 0, 0, 0, ZoneOffset.UTC); + + final Grok grok = compiler.compile("Foo %{DATA:result;date;yyyy.MM.dd} Bar"); + final Match gm = grok.match("Foo 2015.07.31 Bar"); + + assertEquals(1, gm.getMatch().groupCount()); + assertEquals(expectedDate.toInstant(), gm.capture().get("result")); } + @Test + public void test027_datetime_pattern_with_with_hyphens() throws Throwable { + final ZonedDateTime expectedDate = ZonedDateTime.of(2015, 7, 31, 0, 0, 0, 0, ZoneOffset.UTC); + + final Grok grok = compiler.compile("Foo %{DATA:result;date;yyyy-MM-dd} Bar"); + final Match gm = grok.match("Foo 2015-07-31 Bar"); + + assertEquals(1, gm.getMatch().groupCount()); + assertEquals(expectedDate.toInstant(), gm.capture().get("result")); + } @Test + public void allowClassPathPatternFiles() throws Exception { + GrokCompiler compiler = GrokCompiler.newInstance(); + compiler.register(Resources.getResource("patterns/patterns").openStream()); + compiler.compile("%{USERNAME}", false); + } + + @Test(expected = IllegalArgumentException.class) public void createGrokWithDefaultPatterns() throws GrokException { - final Grok grok = Grok.create(); - grok.compile("%{USERNAME}", false); + GrokCompiler compiler = GrokCompiler.newInstance(); + compiler.compile("%{USERNAME}", false); } private void ensureAbortsWithDefinitionMissing(String pattern, String compilePattern, boolean namedOnly) { try { - final Grok grok = Grok.create(ResourceManager.PATTERNS, pattern); - grok.compile(compilePattern, namedOnly); + compiler.compile(pattern); + compiler.compile(compilePattern, namedOnly); fail("should abort due to missing definition"); - } catch (GrokException e) { + } catch (Exception e) { assertThat(e.getMessage(), containsString("No definition for key")); } } + + @Test + public void testGroupTypes() { + Grok grok = compiler.compile( + "%{HTTPDATE:timestamp;date;dd/MMM/yyyy:HH:mm:ss Z} %{USERNAME:username:text} " + + "%{IPORHOST:host}:%{POSINT:port:integer}", + true); + assertEquals(Converter.Type.DATETIME, grok.groupTypes.get("timestamp")); + assertEquals(Converter.Type.STRING, grok.groupTypes.get("username")); + assertEquals(Converter.Type.INT, grok.groupTypes.get("port")); + assertNull(grok.groupTypes.get("host")); + + Match match = grok.match("07/Mar/2004:16:45:56 -0800 test 64.242.88.10:8080"); + Map result = match.capture(); + assertEquals("test", result.get("username")); + assertEquals("64.242.88.10", result.get("host")); + assertEquals(8080, result.get("port")); + assertTrue(result.get("timestamp") instanceof Instant); + } + + @Test + public void testTimeZone() { + // no timezone. default to UTC + String date = "03/19/2018 14:11:00"; + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("MM/dd/yyyy HH:mm:ss"); + Grok grok = compiler.compile("%{DATESTAMP:timestamp;date;MM/dd/yyyy HH:mm:ss}", true); + Instant instant = (Instant) grok.match(date).capture().get("timestamp"); + assertEquals(ZonedDateTime.parse(date, dtf.withZone(ZoneOffset.UTC)).toInstant(), instant); + + // set default timezone to PST + ZoneId pst = ZoneId.of("PST", ZoneId.SHORT_IDS); + grok = compiler.compile("%{DATESTAMP:timestamp;date;MM/dd/yyyy HH:mm:ss}", pst, true); + instant = (Instant) grok.match(date).capture().get("timestamp"); + assertEquals(ZonedDateTime.parse(date, dtf.withZone(pst)).toInstant(), instant); + + // when timestamp has timezone, use it instead of the default. + String dateWithTimeZone = "07/Mar/2004:16:45:56 +0800"; + dtf = DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z"); + grok = compiler.compile("%{HTTPDATE:timestamp;date;dd/MMM/yyyy:HH:mm:ss Z}", pst, true); + instant = (Instant) grok.match(dateWithTimeZone).capture().get("timestamp"); + assertEquals(ZonedDateTime.parse(dateWithTimeZone, dtf.withZone(ZoneOffset.ofHours(8))).toInstant(), instant); + } } diff --git a/src/test/java/io/krakens/grok/api/MessagesTest.java b/src/test/java/io/krakens/grok/api/MessagesTest.java index d78580e..d85b16c 100644 --- a/src/test/java/io/krakens/grok/api/MessagesTest.java +++ b/src/test/java/io/krakens/grok/api/MessagesTest.java @@ -1,35 +1,36 @@ package io.krakens.grok.api; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; +import java.util.Map; import io.krakens.grok.api.exception.GrokException; +import com.google.common.io.Resources; +import org.assertj.core.api.Assertions; import org.junit.Test; public class MessagesTest { - public static final String LOG_FILE = ResourceManager.MESSAGES; - @Test public void test001_linux_messages() throws GrokException, IOException { - Grok grok = new Grok(); - grok.addPatternFromFile(ResourceManager.PATTERNS); - grok.compile("%{MESSAGESLOG}"); + GrokCompiler compiler = GrokCompiler.newInstance(); + compiler.register(Resources.getResource(ResourceManager.PATTERNS).openStream()); + + Grok grok = compiler.compile("%{MESSAGESLOG}"); - BufferedReader br = new BufferedReader(new FileReader(LOG_FILE)); + BufferedReader br = new BufferedReader(new FileReader(Resources.getResource(ResourceManager.MESSAGES).getFile())); String line; System.out.println("Starting test with linux messages log -- may take a while"); while ((line = br.readLine()) != null) { Match gm = grok.match(line); - gm.captures(); - assertNotNull(gm.toJson()); - assertNotEquals("{\"Error\":\"Error\"}", gm.toJson()); + Map map = gm.capture(); + assertNotNull(map); + Assertions.assertThat(map).doesNotContainKey("Error"); } br.close(); } diff --git a/src/test/java/io/krakens/grok/api/ResourceManager.java b/src/test/java/io/krakens/grok/api/ResourceManager.java index cecf168..35d85dc 100644 --- a/src/test/java/io/krakens/grok/api/ResourceManager.java +++ b/src/test/java/io/krakens/grok/api/ResourceManager.java @@ -2,18 +2,14 @@ /** * {@code ResourceManager} . - * - * @author burak aydın. */ public final class ResourceManager { - public static final String PATTERNS = "src/main/resources/patterns/patterns"; + public static final String PATTERNS = "patterns/patterns"; - public static final String MESSAGES = "src/test/resources/message/messages"; + public static final String MESSAGES = "message/messages"; - public static final String ACCESS_LOG = "src/test/resources/access_log"; + public static final String NASA = "nasa/"; - public static final String NASA = "src/test/resources/nasa/"; - - public static final String IP = "src/test/resources/ip"; + public static final String IP = "ip"; }