From 712af2a0ab28332900b745e7c660e15be919cfa4 Mon Sep 17 00:00:00 2001 From: Karen Asmarian Date: Fri, 10 Jan 2025 10:47:18 +0100 Subject: [PATCH] Allow using precise floats in logs --- README.md | 11 ++++++++++ .../json/CompactingJsonBodyFilter.java | 4 ++++ .../json/JacksonJsonFieldBodyFilter.java | 17 ++++++++++++-- .../logbook/json/ParsingJsonCompactor.java | 19 ++++++++++++++-- .../json/PrettyPrintingJsonBodyFilter.java | 15 +++++++++++-- .../json/CompactingJsonBodyFilterTest.java | 13 +++++++++-- .../json/JacksonJsonFieldBodyFilterTest.java | 10 +++++++++ .../json/JsonHttpLogFormatterTest.java | 5 +++-- .../PrettyPrintingJsonBodyFilterTest.java | 22 +++++++++++++++++-- logbook-json/src/test/resources/student.json | 5 +++-- 10 files changed, 107 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3b8142d94..e858ab9ea 100644 --- a/README.md +++ b/README.md @@ -534,6 +534,17 @@ a JSON response body will **not** be escaped and represented as a string: } ``` + +> [!NOTE] +> Logbook is using [BodyFilters](#Filtering) to inline json payload or to find fields for obfuscation. +> Filters for JSON bodies are using Jackson, which comes with a defect of dropping off precision from floating point +> numbers (see [FasterXML/jackson-core/issues/984](https://github.com/FasterXML/jackson-core/issues/984)). +> +> This can be changed by setting the `usePreciseFloats` flag to true in the filter respective filters. +> +> E.g. `new CompactingJsonBodyFilter(true)` will keep the precision of floating point numbers. + + ##### Common Log Format The Common Log Format ([CLF](https://httpd.apache.org/docs/trunk/logs.html#common)) is a standardized text file format used by web servers when generating server log files. The format is supported via diff --git a/logbook-json/src/main/java/org/zalando/logbook/json/CompactingJsonBodyFilter.java b/logbook-json/src/main/java/org/zalando/logbook/json/CompactingJsonBodyFilter.java index a437877ae..d103a332a 100644 --- a/logbook-json/src/main/java/org/zalando/logbook/json/CompactingJsonBodyFilter.java +++ b/logbook-json/src/main/java/org/zalando/logbook/json/CompactingJsonBodyFilter.java @@ -21,6 +21,10 @@ public final class CompactingJsonBodyFilter implements BodyFilter { private final JsonCompactor compactor; + public CompactingJsonBodyFilter(final boolean usePreciseFloats) { + this(new ParsingJsonCompactor(usePreciseFloats)); + } + public CompactingJsonBodyFilter() { this(new ParsingJsonCompactor()); } diff --git a/logbook-json/src/main/java/org/zalando/logbook/json/JacksonJsonFieldBodyFilter.java b/logbook-json/src/main/java/org/zalando/logbook/json/JacksonJsonFieldBodyFilter.java index 1dc10c3a7..1af46ae00 100644 --- a/logbook-json/src/main/java/org/zalando/logbook/json/JacksonJsonFieldBodyFilter.java +++ b/logbook-json/src/main/java/org/zalando/logbook/json/JacksonJsonFieldBodyFilter.java @@ -29,11 +29,20 @@ public class JacksonJsonFieldBodyFilter implements BodyFilter { private final String replacement; private final Set fields; private final JsonFactory factory; + private final boolean usePreciseFloats; - public JacksonJsonFieldBodyFilter(final Collection fieldNames, final String replacement, final JsonFactory factory) { + public JacksonJsonFieldBodyFilter(final Collection fieldNames, + final String replacement, + final JsonFactory factory, + final boolean usePreciseFloats) { this.fields = new HashSet<>(fieldNames); // thread safe for reading this.replacement = replacement; this.factory = factory; + this.usePreciseFloats = usePreciseFloats; + } + + public JacksonJsonFieldBodyFilter(final Collection fieldNames, final String replacement, final JsonFactory factory) { + this(fieldNames, replacement, factory, false); } public JacksonJsonFieldBodyFilter(final Collection fieldNames, final String replacement) { @@ -54,7 +63,11 @@ public String filter(final String body) { JsonToken nextToken; while ((nextToken = parser.nextToken()) != null) { - generator.copyCurrentEvent(parser); + if (usePreciseFloats) { + generator.copyCurrentEventExact(parser); + } else { + generator.copyCurrentEvent(parser); + } if (nextToken == JsonToken.FIELD_NAME && fields.contains(parser.currentName())) { nextToken = parser.nextToken(); generator.writeString(replacement); diff --git a/logbook-json/src/main/java/org/zalando/logbook/json/ParsingJsonCompactor.java b/logbook-json/src/main/java/org/zalando/logbook/json/ParsingJsonCompactor.java index 119bc4f11..d5b1ccd1d 100644 --- a/logbook-json/src/main/java/org/zalando/logbook/json/ParsingJsonCompactor.java +++ b/logbook-json/src/main/java/org/zalando/logbook/json/ParsingJsonCompactor.java @@ -11,12 +11,23 @@ final class ParsingJsonCompactor implements JsonCompactor { private final JsonFactory factory; + private final boolean usePreciseFloats; + + public ParsingJsonCompactor(final JsonFactory factory, final boolean usePreciseFloats) { + this.factory = factory; + this.usePreciseFloats = usePreciseFloats; + } + + public ParsingJsonCompactor(final boolean usePreciseFloats) { + this(new JsonFactory(), usePreciseFloats); + } + public ParsingJsonCompactor() { this(new JsonFactory()); } public ParsingJsonCompactor(final JsonFactory factory) { - this.factory = factory; + this(factory, false); } @Override @@ -27,7 +38,11 @@ public String compact(final String json) throws IOException { final JsonGenerator generator = factory.createGenerator(output)) { while (parser.nextToken() != null) { - generator.copyCurrentEvent(parser); + if (usePreciseFloats) { + generator.copyCurrentEventExact(parser); + } else { + generator.copyCurrentEvent(parser); + } } generator.flush(); diff --git a/logbook-json/src/main/java/org/zalando/logbook/json/PrettyPrintingJsonBodyFilter.java b/logbook-json/src/main/java/org/zalando/logbook/json/PrettyPrintingJsonBodyFilter.java index 059a620a2..f02168059 100644 --- a/logbook-json/src/main/java/org/zalando/logbook/json/PrettyPrintingJsonBodyFilter.java +++ b/logbook-json/src/main/java/org/zalando/logbook/json/PrettyPrintingJsonBodyFilter.java @@ -20,9 +20,16 @@ public final class PrettyPrintingJsonBodyFilter implements BodyFilter { private final JsonFactory factory; + private final boolean usePreciseFloats; - public PrettyPrintingJsonBodyFilter(final JsonFactory factory) { + public PrettyPrintingJsonBodyFilter(final JsonFactory factory, + final boolean usePreciseFloats) { this.factory = factory; + this.usePreciseFloats = usePreciseFloats; + } + + public PrettyPrintingJsonBodyFilter(final JsonFactory factory) { + this(factory, false); } public PrettyPrintingJsonBodyFilter() { @@ -52,7 +59,11 @@ public String filter(@Nullable final String contentType, final String body) { generator.useDefaultPrettyPrinter(); while (parser.nextToken() != null) { - generator.copyCurrentEvent(parser); + if (usePreciseFloats) { + generator.copyCurrentEventExact(parser); + } else { + generator.copyCurrentEvent(parser); + } } generator.flush(); diff --git a/logbook-json/src/test/java/org/zalando/logbook/json/CompactingJsonBodyFilterTest.java b/logbook-json/src/test/java/org/zalando/logbook/json/CompactingJsonBodyFilterTest.java index 3657d7bac..859f9efaa 100644 --- a/logbook-json/src/test/java/org/zalando/logbook/json/CompactingJsonBodyFilterTest.java +++ b/logbook-json/src/test/java/org/zalando/logbook/json/CompactingJsonBodyFilterTest.java @@ -12,12 +12,14 @@ class CompactingJsonBodyFilterTest { /*language=JSON*/ private final String pretty = "{\n" + " \"root\": {\n" + - " \"child\": \"text\"\n" + + " \"child\": \"text\",\n" + + " \"float_child\" : 0.40000000000000002" + " }\n" + "}"; /*language=JSON*/ - private final String compacted = "{\"root\":{\"child\":\"text\"}}"; + private final String compacted = "{\"root\":{\"child\":\"text\",\"float_child\":0.4}}"; + private final String compactedWithPreciseFloat = "{\"root\":{\"child\":\"text\",\"float_child\":0.40000000000000002}}"; @Test void shouldIgnoreEmptyBody() { @@ -50,6 +52,13 @@ void shouldTransformValidJsonRequestWithCompatibleContentType() { assertThat(filtered).isEqualTo(compacted); } + @Test + void shouldPreserveBigFloatOnCopy() { + final String filtered = new CompactingJsonBodyFilter(true) + .filter("application/custom+json", pretty); + assertThat(filtered).isEqualTo(compactedWithPreciseFloat); + } + @Test void shouldSkipInvalidJsonLookingLikeAValidOne() { final String invalidJson = "{invalid}"; diff --git a/logbook-json/src/test/java/org/zalando/logbook/json/JacksonJsonFieldBodyFilterTest.java b/logbook-json/src/test/java/org/zalando/logbook/json/JacksonJsonFieldBodyFilterTest.java index 22e9743d4..08836d6bb 100644 --- a/logbook-json/src/test/java/org/zalando/logbook/json/JacksonJsonFieldBodyFilterTest.java +++ b/logbook-json/src/test/java/org/zalando/logbook/json/JacksonJsonFieldBodyFilterTest.java @@ -1,5 +1,6 @@ package org.zalando.logbook.json; +import com.fasterxml.jackson.core.JsonFactory; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -7,6 +8,7 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -75,6 +77,14 @@ public void doesNotFilterNonJson() throws Exception { assertThat(filtered).contains("Ford"); } + @Test + public void shouldPreserveBigFloatOnCopy() throws Exception { + final String string = getResource("/student.json").trim(); + final JacksonJsonFieldBodyFilter filter = new JacksonJsonFieldBodyFilter(Collections.emptyList(), "XXX", new JsonFactory(), true); + final String filtered = filter.filter("application/json", string); + assertThat(filtered).contains("\"debt\":123450.40000000000000002"); + } + private String getResource(final String path) throws IOException { final byte[] bytes = Files.readAllBytes(Paths.get("src/test/resources/" + path)); return new String(bytes, UTF_8); diff --git a/logbook-json/src/test/java/org/zalando/logbook/json/JsonHttpLogFormatterTest.java b/logbook-json/src/test/java/org/zalando/logbook/json/JsonHttpLogFormatterTest.java index a74248921..65863045c 100644 --- a/logbook-json/src/test/java/org/zalando/logbook/json/JsonHttpLogFormatterTest.java +++ b/logbook-json/src/test/java/org/zalando/logbook/json/JsonHttpLogFormatterTest.java @@ -204,12 +204,13 @@ void shouldNotEmbedReplacedJsonRequestBody(final HttpLogFormatter unit) throws I void shouldEmbedCustomJsonRequestBody(final HttpLogFormatter unit) throws IOException { final HttpRequest request = MockHttpRequest.create() .withContentType("application/custom+json") - .withBodyAsString("{\"name\":\"Bob\"}"); + .withBodyAsString("{\"name\":\"Bob\", \"float_value\": 0.40000000000000002 }"); final String json = unit.format(new SimplePrecorrelation("", systemUTC()), request); with(json) - .assertEquals("$.body.name", "Bob"); + .assertEquals("$.body.name", "Bob") + .assertEquals("$.body.float_value", 0.40000000000000002); } @ParameterizedTest diff --git a/logbook-json/src/test/java/org/zalando/logbook/json/PrettyPrintingJsonBodyFilterTest.java b/logbook-json/src/test/java/org/zalando/logbook/json/PrettyPrintingJsonBodyFilterTest.java index cc23bfa2a..900d7e00f 100644 --- a/logbook-json/src/test/java/org/zalando/logbook/json/PrettyPrintingJsonBodyFilterTest.java +++ b/logbook-json/src/test/java/org/zalando/logbook/json/PrettyPrintingJsonBodyFilterTest.java @@ -1,5 +1,6 @@ package org.zalando.logbook.json; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.zalando.logbook.BodyFilter; @@ -16,13 +17,23 @@ class PrettyPrintingJsonBodyFilterTest { private final String pretty = Stream.of( "{", " \"root\" : {", - " \"child\" : \"text\"", + " \"child\" : \"text\",", + " \"float_child\" : 0.4", + " }", + "}" + ).collect(Collectors.joining(System.lineSeparator())); + + private final String compactedWithPreciseFloat = Stream.of( + "{", + " \"root\" : {", + " \"child\" : \"text\",", + " \"float_child\" : 0.40000000000000002", " }", "}" ).collect(Collectors.joining(System.lineSeparator())); /*language=JSON*/ - private final String compacted = "{\"root\":{\"child\":\"text\"}}"; + private final String compacted = "{\"root\":{\"child\":\"text\", \"float_child\": 0.40000000000000002 }}"; @Test void shouldIgnoreEmptyBody() { @@ -68,4 +79,11 @@ void shouldConstructFromObjectMapper() { assertThat(filtered).isEqualTo(pretty); } + @Test + void shouldPreserveBigFloatOnCopy() { + final String filtered = new PrettyPrintingJsonBodyFilter(new JsonFactory(), true) + .filter("application/json", compacted); + assertThat(filtered).isEqualTo(compactedWithPreciseFloat); + } + } diff --git a/logbook-json/src/test/resources/student.json b/logbook-json/src/test/resources/student.json index 9935be1b8..7fffcf7ec 100644 --- a/logbook-json/src/test/resources/student.json +++ b/logbook-json/src/test/resources/student.json @@ -20,5 +20,6 @@ "Science": 1.9, "PE": 4.0 }, - "nickname": null -} \ No newline at end of file + "nickname": null, + "debt": 123450.40000000000000002 +}