From 1211692e44f5a519c76270fa30b161497e06c7ee Mon Sep 17 00:00:00 2001 From: coryhh <44330697+coryhh@users.noreply.github.com> Date: Tue, 5 Dec 2023 20:05:06 +0800 Subject: [PATCH] fix: support the scenario where the "arex_root" node is decompressed and becomes non-json (#42) --- arex-compare-core/pom.xml | 2 +- .../diff/handler/parse/ObjectParse.java | 46 +++++----- .../diff/handler/parse/sqlparse/SqlParse.java | 2 +- .../handler/verify/VerifyObjectParse.java | 23 +++++ .../arextest/diff/model/CompareBuilder.java | 30 +++++- .../diff/utils/DataBaseCompareUtil.java | 18 +++- .../diff/utils/NormalCompareUtil.java | 18 +++- .../arextest/diff/sdk/CompareProblemTest.java | 91 +++++++++++++++++++ pom.xml | 2 +- 9 files changed, 200 insertions(+), 32 deletions(-) create mode 100644 arex-compare-core/src/main/java/com/arextest/diff/handler/verify/VerifyObjectParse.java diff --git a/arex-compare-core/pom.xml b/arex-compare-core/pom.xml index fedfcc4..b66bebe 100644 --- a/arex-compare-core/pom.xml +++ b/arex-compare-core/pom.xml @@ -5,7 +5,7 @@ arex-compare-parent com.arextest - 0.2.1 + 0.2.2 4.0.0 diff --git a/arex-compare-core/src/main/java/com/arextest/diff/handler/parse/ObjectParse.java b/arex-compare-core/src/main/java/com/arextest/diff/handler/parse/ObjectParse.java index 299bf3c..bef4e76 100644 --- a/arex-compare-core/src/main/java/com/arextest/diff/handler/parse/ObjectParse.java +++ b/arex-compare-core/src/main/java/com/arextest/diff/handler/parse/ObjectParse.java @@ -14,7 +14,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Callable; -import org.apache.commons.lang3.tuple.MutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,47 +32,46 @@ public MsgObjCombination doHandler(RulesConfig rulesConfig) throws Exception { obj1 = TaskThreadFactory.jsonObjectThreadPool.submit(callable1).get(); obj2 = TaskThreadFactory.jsonObjectThreadPool.submit(callable2).get(); - MutablePair objectObjectMutablePair = compatibleDiffType(obj1, obj2); - response.setBaseObj(objectObjectMutablePair.getLeft()); - response.setTestObj(objectObjectMutablePair.getRight()); + response.setBaseObj(obj1); + response.setTestObj(obj2); return response; } - public Object msgToObj(String msg, RulesConfig rulesConfig) throws JsonProcessingException { - Object obj = null; + public Object msgToObj(String msg, RulesConfig rulesConfig) { if (StringUtil.isEmpty(msg)) { - return obj; + return msg; } + Object obj = null; // process the msg String pluginJarUrl = rulesConfig.getPluginJarUrl(); Map, DecompressConfig> decompressConfigMap = rulesConfig.getDecompressConfigMap(); if (decompressConfigMap != null && decompressConfigMap.containsKey(Constant.ROOT_PATH)) { try { - msg = DecompressUtil.decompressPlugin(pluginJarUrl, + String decompressMsg = DecompressUtil.decompressPlugin(pluginJarUrl, decompressConfigMap.get(Constant.ROOT_PATH), msg); + if (!StringUtil.isEmpty(decompressMsg)) { + msg = decompressMsg; + } else { + LOGGER.error("decompress root error"); + } } catch (Throwable throwable) { - LOGGER.error("decompress root error, msg:{}", msg, throwable); + LOGGER.error("decompress root error", throwable); } } - if (msg.startsWith("[")) { - obj = JacksonHelperUtil.objectMapper.readValue(msg, ArrayNode.class); - } else { - obj = JacksonHelperUtil.objectMapper.readValue(msg, ObjectNode.class); + try { + if (msg.startsWith("[")) { + obj = JacksonHelperUtil.objectMapper.readValue(msg, ArrayNode.class); + } else if (msg.startsWith("{")) { + obj = JacksonHelperUtil.objectMapper.readValue(msg, ObjectNode.class); + } else { + obj = msg; + } + } catch (RuntimeException | JsonProcessingException e) { + obj = msg; } return obj; } - - private MutablePair compatibleDiffType(Object obj1, Object obj2) - throws Exception { - MutablePair result = new MutablePair<>(); - if (obj1 == null || obj2 == null || !obj1.getClass().equals(obj2.getClass())) { - throw new Exception("The JSON types corresponding to baseMsg and testMsg are inconsistent."); - } - result.setLeft(obj1); - result.setRight(obj2); - return result; - } } diff --git a/arex-compare-core/src/main/java/com/arextest/diff/handler/parse/sqlparse/SqlParse.java b/arex-compare-core/src/main/java/com/arextest/diff/handler/parse/sqlparse/SqlParse.java index cca5ea7..a6355f2 100644 --- a/arex-compare-core/src/main/java/com/arextest/diff/handler/parse/sqlparse/SqlParse.java +++ b/arex-compare-core/src/main/java/com/arextest/diff/handler/parse/sqlparse/SqlParse.java @@ -96,7 +96,7 @@ public ParsedResult sqlParse(ObjectNode jsonObj, boolean nameToLower) { successParse = false; } } catch (Throwable throwable) { - logger.error("sql parse error", throwable); + logger.warn("sql parse error", throwable); successParse = false; } diff --git a/arex-compare-core/src/main/java/com/arextest/diff/handler/verify/VerifyObjectParse.java b/arex-compare-core/src/main/java/com/arextest/diff/handler/verify/VerifyObjectParse.java new file mode 100644 index 0000000..d786dab --- /dev/null +++ b/arex-compare-core/src/main/java/com/arextest/diff/handler/verify/VerifyObjectParse.java @@ -0,0 +1,23 @@ +package com.arextest.diff.handler.verify; + +import com.arextest.diff.model.parse.MsgObjCombination; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class VerifyObjectParse { + + public boolean verify(MsgObjCombination msgObjCombination) { + Object baseObj = msgObjCombination.getBaseObj(); + Object testObj = msgObjCombination.getTestObj(); + + if (baseObj instanceof ObjectNode && testObj instanceof ObjectNode) { + return true; + } + + if (baseObj instanceof ArrayNode && testObj instanceof ArrayNode) { + return true; + } + return false; + } + +} diff --git a/arex-compare-core/src/main/java/com/arextest/diff/model/CompareBuilder.java b/arex-compare-core/src/main/java/com/arextest/diff/model/CompareBuilder.java index a6437e2..ba68ac9 100644 --- a/arex-compare-core/src/main/java/com/arextest/diff/model/CompareBuilder.java +++ b/arex-compare-core/src/main/java/com/arextest/diff/model/CompareBuilder.java @@ -10,6 +10,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.Future; public class CompareBuilder { @@ -90,6 +91,28 @@ public CompareBuilder noDiff(String baseMsg, String testMsg) { return this; } + public CompareBuilder addEqualsCompare(Object baseMsg, Object testMsg, RulesConfig rulesConfig) { + String baseMsgStr = baseMsg == null ? null : baseMsg.toString(); + String testMsgStr = testMsg == null ? null : testMsg.toString(); + boolean equals = Objects.equals(baseMsgStr, testMsgStr); + this.code(equals ? DiffResultCode.COMPARED_WITHOUT_DIFFERENCE + : DiffResultCode.COMPARED_WITH_DIFFERENCE); + this.message("compare successfully"); + this.msgInfo(baseMsgStr, testMsgStr); + this.processedBaseMsg(rulesConfig.isQuickCompare() ? rulesConfig.getBaseMsg() : baseMsgStr); + this.processedTestMsg(rulesConfig.isQuickCompare() ? rulesConfig.getTestMsg() : testMsgStr); + if (!equals && !rulesConfig.isQuickCompare()) { + LogEntity logEntity = new LogEntity(); + logEntity.setBaseValue(baseMsgStr); + logEntity.setTestValue(testMsgStr); + UnmatchedPairEntity pairEntity = new UnmatchedPairEntity(); + pairEntity.setUnmatchedType(UnmatchedType.UNMATCHED); + logEntity.setPathPair(pairEntity); + this.logs(Collections.singletonList(logEntity)); + } + return this; + } + public CompareBuilder addStringUnMatched(String baseMsg, String testMsg) { this.code(DiffResultCode.COMPARED_WITH_DIFFERENCE); this.message("compare successfully"); @@ -140,8 +163,9 @@ public CompareBuilder exception(String baseMsg, String testMsg, String remark) { return this; } + // the endpoint of the logic "quick compare" public CompareBuilder addFindErrorException(String baseMsg, String testMsg, - List> processedMsgList, Exception e) { + List> processedMsgList, Exception e, RulesConfig rulesConfig) { String processedBaseMsg; String processedTestMsg; try { @@ -155,8 +179,8 @@ public CompareBuilder addFindErrorException(String baseMsg, String testMsg, .code(DiffResultCode.COMPARED_WITH_DIFFERENCE) .message("compare successfully") .msgInfo(baseMsg, testMsg) - .processedBaseMsg(processedBaseMsg) - .processedTestMsg(processedTestMsg); + .processedBaseMsg(rulesConfig.isQuickCompare() ? baseMsg : processedBaseMsg) + .processedTestMsg(rulesConfig.isQuickCompare() ? testMsg : processedTestMsg); } private String exceptionToString(Throwable e) { diff --git a/arex-compare-core/src/main/java/com/arextest/diff/utils/DataBaseCompareUtil.java b/arex-compare-core/src/main/java/com/arextest/diff/utils/DataBaseCompareUtil.java index 1f8c5cb..c9f5689 100644 --- a/arex-compare-core/src/main/java/com/arextest/diff/utils/DataBaseCompareUtil.java +++ b/arex-compare-core/src/main/java/com/arextest/diff/utils/DataBaseCompareUtil.java @@ -15,6 +15,7 @@ import com.arextest.diff.handler.parse.JSONStructureParse; import com.arextest.diff.handler.parse.ObjectParse; import com.arextest.diff.handler.parse.sqlparse.SqlParse; +import com.arextest.diff.handler.verify.VerifyObjectParse; import com.arextest.diff.model.CompareResult; import com.arextest.diff.model.RulesConfig; import com.arextest.diff.model.enumeration.DiffResultCode; @@ -42,6 +43,8 @@ public class DataBaseCompareUtil { private static ObjectParse objectParse = new ObjectParse(); + private static VerifyObjectParse verifyObjectParse = new VerifyObjectParse(); + private static SqlParse sqlParse = new SqlParse(); private static JSONParse jsonParse = new JSONParse(); @@ -71,6 +74,18 @@ public CompareResult jsonCompare(RulesConfig rulesConfig) { timeConsumerWatch.start(TimeMetricLabel.OBJECT_PARSE); msgObjCombination = objectParse.doHandler(rulesConfig); timeConsumerWatch.end(TimeMetricLabel.OBJECT_PARSE); + + // verify parse results + boolean verifyResult = verifyObjectParse.verify(msgObjCombination); + if (!verifyResult) { + result = CompareResult.builder() + .addEqualsCompare(msgObjCombination.getBaseObj(), msgObjCombination.getTestObj(), + rulesConfig) + .build(); + timeConsumerWatch.end(TimeMetricLabel.TOTAL); + timeConsumerWatch.record(result); + return result; + } } catch (Exception e) { result = CompareResult.builder().addStringUnMatched(baseMsg, testMsg).build(); timeConsumerWatch.end(TimeMetricLabel.TOTAL); @@ -145,7 +160,8 @@ public CompareResult jsonCompare(RulesConfig rulesConfig) { } catch (SelectIgnoreException e) { result = CompareResult.builder().noDiff(baseMsg, testMsg).build(); } catch (FindErrorException e) { - result = CompareResult.builder().addFindErrorException(baseMsg, testMsg, processedMsgList, e) + result = CompareResult.builder() + .addFindErrorException(baseMsg, testMsg, processedMsgList, e, rulesConfig) .build(); } catch (Exception e) { LOGGER.error("compare error, exception:", e); diff --git a/arex-compare-core/src/main/java/com/arextest/diff/utils/NormalCompareUtil.java b/arex-compare-core/src/main/java/com/arextest/diff/utils/NormalCompareUtil.java index 000dde0..e5b356e 100644 --- a/arex-compare-core/src/main/java/com/arextest/diff/utils/NormalCompareUtil.java +++ b/arex-compare-core/src/main/java/com/arextest/diff/utils/NormalCompareUtil.java @@ -13,6 +13,7 @@ import com.arextest.diff.handler.parse.JSONParse; import com.arextest.diff.handler.parse.JSONStructureParse; import com.arextest.diff.handler.parse.ObjectParse; +import com.arextest.diff.handler.verify.VerifyObjectParse; import com.arextest.diff.model.CompareResult; import com.arextest.diff.model.RulesConfig; import com.arextest.diff.model.enumeration.DiffResultCode; @@ -40,6 +41,8 @@ public class NormalCompareUtil { private static ObjectParse objectParse = new ObjectParse(); + private static VerifyObjectParse verifyObjectParse = new VerifyObjectParse(); + private static JSONParse jsonParse = new JSONParse(); private static FillResultSync fillResultSync = new FillResultSync(); @@ -66,6 +69,18 @@ public CompareResult jsonCompare(RulesConfig rulesConfig) { timeConsumerWatch.start(TimeMetricLabel.OBJECT_PARSE); msgObjCombination = objectParse.doHandler(rulesConfig); timeConsumerWatch.end(TimeMetricLabel.OBJECT_PARSE); + + // verify parse results + boolean verifyResult = verifyObjectParse.verify(msgObjCombination); + if (!verifyResult) { + result = CompareResult.builder() + .addEqualsCompare(msgObjCombination.getBaseObj(), msgObjCombination.getTestObj(), + rulesConfig) + .build(); + timeConsumerWatch.end(TimeMetricLabel.TOTAL); + timeConsumerWatch.record(result); + return result; + } } catch (Exception e) { result = CompareResult.builder().addStringUnMatched(baseMsg, testMsg).build(); timeConsumerWatch.end(TimeMetricLabel.TOTAL); @@ -132,7 +147,8 @@ public CompareResult jsonCompare(RulesConfig rulesConfig) { .build(); } catch (FindErrorException e) { - result = CompareResult.builder().addFindErrorException(baseMsg, testMsg, processedMsgList, e) + result = CompareResult.builder() + .addFindErrorException(baseMsg, testMsg, processedMsgList, e, rulesConfig) .build(); } catch (Exception e) { LOGGER.error("compare error, exception:", e); diff --git a/arex-compare-core/src/test/java/com/arextest/diff/sdk/CompareProblemTest.java b/arex-compare-core/src/test/java/com/arextest/diff/sdk/CompareProblemTest.java index c5f3b50..78f10fb 100644 --- a/arex-compare-core/src/test/java/com/arextest/diff/sdk/CompareProblemTest.java +++ b/arex-compare-core/src/test/java/com/arextest/diff/sdk/CompareProblemTest.java @@ -2,6 +2,7 @@ import com.arextest.diff.model.CompareOptions; import com.arextest.diff.model.CompareResult; +import com.arextest.diff.model.DecompressConfig; import com.arextest.diff.model.enumeration.CategoryType; import com.arextest.diff.model.enumeration.DiffResultCode; import java.util.Arrays; @@ -409,4 +410,94 @@ public void testNullEqualsEmptyProblem() { Assert.assertEquals(result.getLogs().size(), 0); } + @Test + public void testNullString() { + CompareSDK sdk = new CompareSDK(); + sdk.getGlobalOptions().putNameToLower(true).putNullEqualsEmpty(true); + String baseMsg = null; + String testMsg = ""; + CompareResult result = sdk.compare(baseMsg, testMsg); + Assert.assertEquals(result.getCode(), 0); + } + + @Test + public void testRootDecompress() { + CompareSDK sdk = new CompareSDK(); + sdk.getGlobalOptions().putNameToLower(true).putNullEqualsEmpty(true) + .putPluginJarUrl("./lib/arex-compare-sdk-plugin-0.1.0-jar-with-dependencies.jar"); + + CompareOptions compareOptions = CompareOptions.options() + .putDecompressConfig(new DecompressConfig("Gzip", + Arrays.asList(Arrays.asList("arex_root")))); + + String baseMsg = "H4sIAAAAAAAAAEtMTAQALXMH8AMAAAA="; + String testMsg = "H4sIAAAAAAAAAEtMTEwEAEXlmK0EAAAA"; + CompareResult result = sdk.compare(baseMsg, testMsg, compareOptions); + Assert.assertEquals("aaa", result.getProcessedBaseMsg()); + + CompareResult quickResult = sdk.quickCompare(baseMsg, testMsg, compareOptions); + Assert.assertEquals("H4sIAAAAAAAAAEtMTAQALXMH8AMAAAA=", quickResult.getProcessedBaseMsg()); + } + + @Test + public void testRootDecompress2() { + CompareSDK sdk = new CompareSDK(); + sdk.getGlobalOptions().putNameToLower(true).putNullEqualsEmpty(true) + .putPluginJarUrl("./lib/arex-compare-sdk-plugin-0.1.0-jar-with-dependencies.jar"); + + CompareOptions compareOptions = CompareOptions.options() + .putDecompressConfig(new DecompressConfig("Gzip", + Arrays.asList(Arrays.asList("arex_root")))); + + String baseMsg = null; + String testMsg = "H4sIAAAAAAAAAEtMTEwEAEXlmK0EAAAA"; + CompareResult result = sdk.compare(baseMsg, testMsg, compareOptions); + Assert.assertEquals("aaaa", result.getProcessedTestMsg()); + + CompareResult quickResult = sdk.quickCompare(baseMsg, testMsg, compareOptions); + Assert.assertEquals("H4sIAAAAAAAAAEtMTEwEAEXlmK0EAAAA=", quickResult.getProcessedTestMsg()); + + } + + @Test + public void testRootDecompress3() { + CompareSDK sdk = new CompareSDK(); + sdk.getGlobalOptions().putNameToLower(true).putNullEqualsEmpty(true) + .putPluginJarUrl("./lib/arex-compare-sdk-plugin-0.1.0-jar-with-dependencies.jar"); + + CompareOptions compareOptions = CompareOptions.options() + .putDecompressConfig(new DecompressConfig("Gzip", + Arrays.asList(Arrays.asList("arex_root")))); + + String baseMsg = "H4sIAAAAAAAAAKtWSlSyUkpSqgUAnFz2awkAAAA="; + String testMsg = "H4sIAAAAAAAAAKtWSlSyAuJaAMXisGkJAAAA"; + CompareResult result = sdk.compare(baseMsg, testMsg, compareOptions); + Assert.assertEquals(1, result.getCode()); + + CompareResult quickResult = sdk.quickCompare(baseMsg, testMsg, compareOptions); + Assert.assertEquals("H4sIAAAAAAAAAKtWSlSyAuJaAMXisGkJAAAA", quickResult.getProcessedTestMsg()); + + } + + @Test + public void testRootDecompress4() { + CompareSDK sdk = new CompareSDK(); + sdk.getGlobalOptions().putNameToLower(true).putNullEqualsEmpty(true) + .putPluginJarUrl("./lib/arex-compare-sdk-plugin-0.1.0-jar-with-dependencies.jar"); + + CompareOptions compareOptions = CompareOptions.options() + .putDecompressConfig(new DecompressConfig("Gzip", + Arrays.asList(Arrays.asList("arex_root")))); + + String baseMsg = "H4sIAAAAAAAAAKtOTEwEAL5SPJIEAAAA"; + String testMsg = "H4sIAAAAAAAAAKtOBAIAmhz8xAUAAAA="; + CompareResult result = sdk.compare(baseMsg, testMsg, compareOptions); + Assert.assertEquals(1, result.getCode()); + + CompareResult quickResult = sdk.quickCompare(baseMsg, testMsg, compareOptions); + Assert.assertEquals("H4sIAAAAAAAAAKtOBAIAmhz8xAUAAAA=", quickResult.getProcessedTestMsg()); + + } + + } diff --git a/pom.xml b/pom.xml index b23b479..5addbd1 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ com.arextest arex-compare-parent pom - 0.2.1 + 0.2.2 arex-compare-extension arex-compare-core