Add option unicode

perlpunk · Jan 3, 2024 · 6ec565d · 6ec565d
1 parent 79511dd
commit 6ec565d
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 19 deletions.
diff --git a/LibYAML/XS.xs b/LibYAML/XS.xs
@@ -124,6 +124,7 @@ emit_string_events(AV *perl_events, HV *options)
         yaml_emitter_t emitter;
         SV **val;
         SV *yaml = newSVpvn("", 0);
+        int unicode = 1;
 
         XCPT_TRY_START
         {
@@ -140,7 +141,14 @@ emit_string_events(AV *perl_events, HV *options)
             }
             yaml_emitter_set_output(&emitter, &append_output, (void *) yaml);
             yaml_emitter_set_canonical(&emitter, 0);
-            yaml_emitter_set_unicode(&emitter, 0);
+            yaml_emitter_set_unicode(&emitter, 1);
+            val = hv_fetch(options, "unicode", 7, TRUE);
+            if (val && SvOK(*val) && SvTRUE(*val)) {
+                unicode = 1;
+            }
+            else if (val && SvOK(*val) && ! SvTRUE(*val)) {
+                unicode = 0;
+            }
 
             emit_events(&emitter, perl_events);
 
@@ -155,7 +163,13 @@ emit_string_events(AV *perl_events, HV *options)
         }
 
         if (yaml) {
-            SvUTF8_off(yaml);
+            if (unicode) {
+                SvUTF8_off(yaml);
+            }
+            else {
+                (void)sv_utf8_decode(yaml);
+                SvUTF8_on(yaml);
+            }
         }
         RETVAL = yaml;
 
@@ -188,7 +202,7 @@ emit_file_events(const char *filename, AV *perl_events, HV *options)
             output = fopen(filename, "wb");
             yaml_emitter_set_output_file(&emitter, output);
             yaml_emitter_set_canonical(&emitter, 0);
-            yaml_emitter_set_unicode(&emitter, 0);
+            yaml_emitter_set_unicode(&emitter, 1);
 
             emit_events(&emitter, perl_events);
 
@@ -240,7 +254,7 @@ emit_filehandle_events(FILE *output, AV *perl_events, HV *options)
 
             yaml_emitter_set_output_file(&emitter, output);
             yaml_emitter_set_canonical(&emitter, 0);
-            yaml_emitter_set_unicode(&emitter, 0);
+            yaml_emitter_set_unicode(&emitter, 1);
 
             emit_events(&emitter, perl_events);
 

diff --git a/LibYAML/etc/perl_libyaml.c b/LibYAML/etc/perl_libyaml.c
@@ -324,7 +324,7 @@ perl_to_libyaml_event(yaml_emitter_t *emitter, HV *perl_event)
         }
 
         if (strEQ(type, "stream_start_event")) {
-            ok = yaml_stream_start_event_initialize(&event, 0);
+            ok = yaml_stream_start_event_initialize(&event, YAML_UTF8_ENCODING);
         }
         else if (strEQ(type, "stream_end_event")) {
             ok = yaml_stream_end_event_initialize(&event);

diff --git a/lib/YAML/LibYAML/API.pm b/lib/YAML/LibYAML/API.pm
@@ -159,10 +159,18 @@ YAML::LibYAML::API - Wrapper around the C libyaml library
     YAML::LibYAML::API::XS::parse_filehandle_events($fh, $events);
 
     # emit
-    my $yaml = YAML::LibYAML::API::XS::emit_string_events($events);
+    my $yaml = YAML::LibYAML::API::XS::emit_string_events($events, {
+        indent => 2,
+        width => 80,
+        unicode => 1, # emit unicode
+    });
     # or:
-    YAML::LibYAML::API::XS::emit_file_events($filename, $events);
-    YAML::LibYAML::API::XS::emit_filehandle_events($fh, $events);
+    my $options = {
+        indent => 2,
+        width => 80,
+    };
+    YAML::LibYAML::API::XS::emit_file_events($filename, $events, $options);
+    YAML::LibYAML::API::XS::emit_filehandle_events($fh, $events, $options);
 
 =head1 DESCRIPTION
 

diff --git a/t/10.basic.t b/t/10.basic.t
@@ -197,7 +197,7 @@ my @exp_file_events = (
         start => { line => 1, column => 0 },
         end   => { line => 1, column => 0 },
     },
-    { name => 'scalar_event', value => 'a', style => YAML_PLAIN_SCALAR_STYLE,
+    { name => 'scalar_event', value => decode_utf8('ä'), style => YAML_PLAIN_SCALAR_STYLE,
         start => { line => 1, column => 0 },
         end   => { line => 1, column => 1 },
     },
@@ -256,7 +256,7 @@ subtest emit_file_events => sub {
 
 
     open $fh, ">", "$Bin/data/simple.yaml.out" or die $!;
-    YAML::LibYAML::API::emit_filehandle_events($fh, $ev);
+    YAML::LibYAML::API::emit_filehandle_events($fh, $ev, { unicode => 0 });
     close $fh;
 
     open $fh, "<", "$Bin/data/simple.yaml.out" or die $!;
@@ -267,18 +267,17 @@ subtest emit_file_events => sub {
 
 subtest unicode => sub {
     my $ev = [];
-    $yaml = "- ö";
+    $yaml = "- ä";
+    my $dec = decode_utf8 "ä";
     YAML::LibYAML::API::parse_string_events($yaml, $ev);
     my $value = encode_utf8 $ev->[3]->{value};
-    cmp_ok($value, 'eq', "ö", "utf8 parse");
+    cmp_ok($value, 'eq', "ä", "utf8 parse");
 
-    $ev->[3]->{value} = decode_utf8 "ä";
-    my $dump = YAML::LibYAML::API::emit_string_events($ev);
-    cmp_ok($dump, '=~', qr{- "\\xE4"}i, "utf8 emit");
+    my $dump = YAML::LibYAML::API::emit_string_events($ev, { unicode => 0 });
+    cmp_ok($dump, '=~', qr{- $dec}i, "utf8 decoded emit");
 
-    $ev->[3]->{value} = "\303\274 \303\300";
-    $dump = YAML::LibYAML::API::emit_string_events($ev);
-    cmp_ok($dump, '=~', qr{- "\\xC3\\xBC \\xC3\\xC0"}i, "binary emit");
+    $dump = YAML::LibYAML::API::emit_string_events($ev, { unicode => 1 });
+    cmp_ok($dump, '=~', qr{- ä}i, "utf8 emit");
 };
 
 subtest indent => sub {

diff --git a/t/data/simple.yaml b/t/data/simple.yaml
@@ -1,2 +1,2 @@
 ---
-a: b
+ä: b