From 55617299f2a8e203ffc6816cbce96c7a57416eb6 Mon Sep 17 00:00:00 2001 From: TobiasNx Date: Wed, 18 Sep 2024 09:35:38 +0200 Subject: [PATCH 1/7] Add new fix method to set variable with element value #282 --- .../main/java/org/metafacture/metafix/FixMethod.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java index 4ae2ac32..9a46856a 100644 --- a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -124,6 +124,18 @@ public void apply(final Metafix metafix, final Record record, final List } }, + to_var { + @Override + + public void apply(final Metafix metafix, final Record record, final List params, final Map options) { + final String fieldName = params.get(0); + final String variableName = params.get(1); + final String variableValue = record.get(fieldName).toString(); + + metafix.getVars().put(variableName, variableValue); + } + }, + // RECORD-LEVEL METHODS: add_field { From 726d0fa0efa12bca7f2fe04a060f80fda1958ae7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BClte?= Date: Mon, 23 Sep 2024 15:15:11 +0200 Subject: [PATCH 2/7] Add test for to_var #282 --- .../metafix/MetafixMethodTest.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index 8eca36ae..0ce11530 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -4084,4 +4084,39 @@ public void shouldTransformStringToBase64() { ); } + @Test + public void shouldCreateVariableFromLiteralValue() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_var('data.title','testVar')", + "add_field('testResult','This is a $[testVar]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", "test"); + i.endEntity(); + i.endRecord(); + i.startRecord("2"); + i.startEntity("data"); + i.literal("title", "second-test"); + i.endEntity(); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("title", "test"); + o.get().endEntity(); + o.get().literal("testResult", "This is a test"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().startEntity("data"); + o.get().literal("title", "second-test"); + o.get().endEntity(); + o.get().literal("testResult", "This is a second-test"); + o.get().endRecord(); + } + ); + } + } From a52dc9bd3c897d7c6461ba6846e3b2819faa653b Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Tue, 24 Sep 2024 17:17:05 +0200 Subject: [PATCH 3/7] Account for missing values in `to_var()` Fix function. (#365) Also, test edge cases. --- .../org/metafacture/metafix/FixMethod.java | 9 +--- .../metafix/MetafixMethodTest.java | 42 ++++++++++++++++--- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java index 9a46856a..1811826a 100644 --- a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -123,16 +123,11 @@ public void apply(final Metafix metafix, final Record record, final List metafix.getVars().putAll(options); } }, - to_var { @Override - public void apply(final Metafix metafix, final Record record, final List params, final Map options) { - final String fieldName = params.get(0); - final String variableName = params.get(1); - final String variableValue = record.get(fieldName).toString(); - - metafix.getVars().put(variableName, variableValue); + final Value value = record.get(params.get(0)); + metafix.getVars().put(params.get(1), Value.isNull(value) ? "" : value.toString()); } }, diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index 0ce11530..c69d17b0 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -4084,7 +4084,7 @@ public void shouldTransformStringToBase64() { ); } - @Test + @Test // checkstyle-disable-line JavaNCSS public void shouldCreateVariableFromLiteralValue() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( "to_var('data.title','testVar')", @@ -4098,11 +4098,26 @@ public void shouldCreateVariableFromLiteralValue() { i.endRecord(); i.startRecord("2"); i.startEntity("data"); - i.literal("title", "second-test"); + i.literal("title", "test1"); + i.literal("title", "test2"); + i.endEntity(); + i.endRecord(); + i.startRecord("3"); + i.startEntity("data"); + i.startEntity("title"); + i.literal("key", "value"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + i.startRecord("4"); + i.endRecord(); + i.startRecord("5"); + i.startEntity("data"); + i.literal("title", "final-test"); i.endEntity(); i.endRecord(); }, - o -> { + (o, f) -> { o.get().startRecord("1"); o.get().startEntity("data"); o.get().literal("title", "test"); @@ -4111,9 +4126,26 @@ public void shouldCreateVariableFromLiteralValue() { o.get().endRecord(); o.get().startRecord("2"); o.get().startEntity("data"); - o.get().literal("title", "second-test"); + o.get().literal("title", "test1"); + o.get().literal("title", "test2"); + o.get().endEntity(); + o.get().literal("testResult", "This is a [test1, test2]"); + o.get().endRecord(); + o.get().startRecord("3"); + o.get().startEntity("data"); + o.get().startEntity("title"); + o.get().literal("key", "value"); + f.apply(2).endEntity(); + o.get().literal("testResult", "This is a {key=value}"); + o.get().endRecord(); + o.get().startRecord("4"); + o.get().literal("testResult", "This is a "); + o.get().endRecord(); + o.get().startRecord("5"); + o.get().startEntity("data"); + o.get().literal("title", "final-test"); o.get().endEntity(); - o.get().literal("testResult", "This is a second-test"); + o.get().literal("testResult", "This is a final-test"); o.get().endRecord(); } ); From 46d1ef5a53582f83174ad92347b1385c6bc89532 Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Fri, 11 Oct 2024 16:29:30 +0200 Subject: [PATCH 4/7] Implement optional default value for `to_var()` Fix function. (#365) --- .../org/metafacture/metafix/FixMethod.java | 3 +- .../metafix/MetafixMethodTest.java | 33 +++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java index 1811826a..25e23e97 100644 --- a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -127,7 +127,7 @@ public void apply(final Metafix metafix, final Record record, final List @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final Value value = record.get(params.get(0)); - metafix.getVars().put(params.get(1), Value.isNull(value) ? "" : value.toString()); + metafix.getVars().put(params.get(1), Value.isNull(value) ? options.getOrDefault(DEFAULT_OPTION, "") : value.toString()); } }, @@ -694,6 +694,7 @@ public void apply(final Metafix metafix, final Record record, final List private static final String FILEMAP_SEPARATOR_OPTION = "sep_char"; private static final String FILEMAP_DEFAULT_SEPARATOR = ","; + private static final String DEFAULT_OPTION = "default"; private static final String ERROR_STRING_OPTION = "error_string"; private static final Random RANDOM = new Random(); diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index c69d17b0..9d6925ca 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -4087,8 +4087,8 @@ public void shouldTransformStringToBase64() { @Test // checkstyle-disable-line JavaNCSS public void shouldCreateVariableFromLiteralValue() { MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( - "to_var('data.title','testVar')", - "add_field('testResult','This is a $[testVar]')" + "to_var('data.title', 'testVar')", + "add_field('testResult', 'This is a $[testVar]')" ), i -> { i.startRecord("1"); @@ -4151,4 +4151,33 @@ public void shouldCreateVariableFromLiteralValue() { ); } + @Test + public void shouldCreateVariableFromLiteralValueWithDefault() { + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_var('data.title', 'testVar', 'default': 'n/a')", + "add_field('testResult', 'This is a $[testVar]')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", "test"); + i.endEntity(); + i.endRecord(); + i.startRecord("2"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().startEntity("data"); + o.get().literal("title", "test"); + o.get().endEntity(); + o.get().literal("testResult", "This is a test"); + o.get().endRecord(); + o.get().startRecord("2"); + o.get().literal("testResult", "This is a n/a"); + o.get().endRecord(); + } + ); + } + } From cee638a62aadc2f7ca24e4e45cb1e749d3e0c985 Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Fri, 11 Oct 2024 16:34:55 +0200 Subject: [PATCH 5/7] Only accept string values in `to_var()` Fix function. (#365) --- .../org/metafacture/metafix/FixMethod.java | 2 +- .../metafix/MetafixMethodTest.java | 72 +++++++++++-------- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java index 25e23e97..da8ce7b3 100644 --- a/metafix/src/main/java/org/metafacture/metafix/FixMethod.java +++ b/metafix/src/main/java/org/metafacture/metafix/FixMethod.java @@ -127,7 +127,7 @@ public void apply(final Metafix metafix, final Record record, final List @Override public void apply(final Metafix metafix, final Record record, final List params, final Map options) { final Value value = record.get(params.get(0)); - metafix.getVars().put(params.get(1), Value.isNull(value) ? options.getOrDefault(DEFAULT_OPTION, "") : value.toString()); + metafix.getVars().put(params.get(1), Value.isNull(value) ? options.getOrDefault(DEFAULT_OPTION, "") : value.asString()); } }, diff --git a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java index 9d6925ca..8b167257 100644 --- a/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java +++ b/metafix/src/test/java/org/metafacture/metafix/MetafixMethodTest.java @@ -4097,27 +4097,14 @@ public void shouldCreateVariableFromLiteralValue() { i.endEntity(); i.endRecord(); i.startRecord("2"); - i.startEntity("data"); - i.literal("title", "test1"); - i.literal("title", "test2"); - i.endEntity(); i.endRecord(); i.startRecord("3"); i.startEntity("data"); - i.startEntity("title"); - i.literal("key", "value"); - i.endEntity(); - i.endEntity(); - i.endRecord(); - i.startRecord("4"); - i.endRecord(); - i.startRecord("5"); - i.startEntity("data"); i.literal("title", "final-test"); i.endEntity(); i.endRecord(); }, - (o, f) -> { + o -> { o.get().startRecord("1"); o.get().startEntity("data"); o.get().literal("title", "test"); @@ -4125,23 +4112,9 @@ public void shouldCreateVariableFromLiteralValue() { o.get().literal("testResult", "This is a test"); o.get().endRecord(); o.get().startRecord("2"); - o.get().startEntity("data"); - o.get().literal("title", "test1"); - o.get().literal("title", "test2"); - o.get().endEntity(); - o.get().literal("testResult", "This is a [test1, test2]"); - o.get().endRecord(); - o.get().startRecord("3"); - o.get().startEntity("data"); - o.get().startEntity("title"); - o.get().literal("key", "value"); - f.apply(2).endEntity(); - o.get().literal("testResult", "This is a {key=value}"); - o.get().endRecord(); - o.get().startRecord("4"); o.get().literal("testResult", "This is a "); o.get().endRecord(); - o.get().startRecord("5"); + o.get().startRecord("3"); o.get().startEntity("data"); o.get().literal("title", "final-test"); o.get().endEntity(); @@ -4180,4 +4153,45 @@ public void shouldCreateVariableFromLiteralValueWithDefault() { ); } + @Test + public void shouldNotCreateVariableFromArrayValue() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Array", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_var('data.title', 'testVar')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.literal("title", "test1"); + i.literal("title", "test2"); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + + @Test + public void shouldNotCreateVariableFromHashValue() { + MetafixTestHelpers.assertExecutionException(IllegalStateException.class, "Expected String, got Hash", () -> + MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList( + "to_var('data.title', 'testVar')" + ), + i -> { + i.startRecord("1"); + i.startEntity("data"); + i.startEntity("title"); + i.literal("key", "value"); + i.endEntity(); + i.endEntity(); + i.endRecord(); + }, + o -> { + } + ) + ); + } + } From 8ad9cde866c8b9907c0cfd77a32c1720dcd3054d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20B=C3=BClte?= Date: Mon, 14 Oct 2024 14:52:12 +0200 Subject: [PATCH 6/7] Add documentation for `to_var` function #282 --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 287a9602..69007da8 100644 --- a/README.md +++ b/README.md @@ -267,6 +267,23 @@ put_vars( [Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-fix+path:FixMethod.java+"+put_vars+{") +##### `to_var` + +Defines a single global variable that can be referenced with `$[]` and assinges the value of the ``. + +```perl +to_var("", "") +``` + +Options: + +- `"default"`: provide default value if source field does not exist. + + +[Example in Playground](https://metafacture.org/playground/?example=to_var) + +[Java Code](https://github.com/search?type=code&q=repo:metafacture/metafacture-fix+path:FixMethod.java+"+to_var+{") + #### Record-level functions ##### `add_field` From 93ee08ef9825fe849faa1857fc8fc2766cb4e565 Mon Sep 17 00:00:00 2001 From: TobiasNx <61879957+TobiasNx@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:32:53 +0200 Subject: [PATCH 7/7] Apply suggestions from code review #282 Review by @blackwinter --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 69007da8..4f42f1ce 100644 --- a/README.md +++ b/README.md @@ -269,16 +269,15 @@ put_vars( ##### `to_var` -Defines a single global variable that can be referenced with `$[]` and assinges the value of the ``. +Defines a single global variable that can be referenced with `$[]` and assigns the value of the ``. ```perl -to_var("", "") +to_var("", "") ``` Options: -- `"default"`: provide default value if source field does not exist. - +- `default`: Default value if source field does not exist. The option needs to be written in quotation marks because it is a reserved word in Java. (Default: Empty string) [Example in Playground](https://metafacture.org/playground/?example=to_var)