Add support for full casing mappings.

CHANGES.md

@@ -2,6 +2,8 @@ Version 4.0.0-dev
 -----------------
 - Increase the minimum requirements to C++ 17 [incompatible change].
 - Start using `std::string_view` in the API [incompatible change].
+- Add also full casing mappings (where the casing of a single codepoint
+  is longer than 1 codepoint).
 - Replace the system generating the API documentation.
 - Update Unicode data to 15.1.0.
 

README.md

@@ -8,7 +8,7 @@ It is versioned using [Semantic Versioning](http://semver.org/).
 If currently offers:
 - utf-8 and utf-16 encodings
 - Unicode General Category info
-- simple lowercasing, titlecasing and uppercasing
+- simple and full lower/title/uppercasing
 - Unicode normalization forms
 - efficient stripping of combining marks
 

doc/API.md

@@ -51,6 +51,13 @@ class unicode {
   static inline char32_t lowercase(char32_t chr);
   static inline char32_t uppercase(char32_t chr);
   static inline char32_t titlecase(char32_t chr);
+
+  // Returns the full lowercase/uppercase/titlecase mapping of the given code
+  // point from [Unicode Character Database](http://www.unicode.org/reports/tr44/).
+  // If no full mapping is defined, return the corresponding simple mapping.
+  static std::u32string lowercase_full(char32_t chr);
+  static std::u32string uppercase_full(char32_t chr);
+  static std::u32string titlecase_full(char32_t chr);
 };
 ```
 

gen/generate.pl

@@ -208,6 +208,27 @@ sub split_long {
   $data_ref->{rawdata} = split_long($data_ref->{rawdata});
 }
 
+# Generate code for performing full casing mappings.
+my %full_casings = ();
+open ($f, "<", "$UnicodeDataDir/SpecialCasing.txt") or die "Cannot open '$UnicodeDataDir/SpecialCasing.txt': $!";
+while (<$f>) {
+  chomp;
+  s/\s*(#.*)?$//;
+  next unless length;
+  my @parts = split /\s*;\s*/, $_;
+  @parts > 4 and $parts[4] and next;
+  my @casing = ("LOWER", "TITLE", "UPPER");
+  for (my $i = 0; $i < @casing; $i++) {
+    if ($parts[$i + 1] ne $parts[0] and $parts[$i + 1] =~ /\s/) {
+      $full_casings{$casing[$i]}->{"U'\\u$parts[0]'"} = 'U"' . join("", map {"\\u$_"} split(/\s+/, $parts[$i + 1])) . '"';
+    }
+  }
+}
+close $f;
+foreach my $casing (keys %full_casings) {
+  $full_casings{$casing} = join("\n    ", map {"case $_: return $full_casings{$casing}->{$_};"} sort(keys %{$full_casings{$casing}}));
+}
+
 # Replace templates in given files.
 while (@ARGV) {
   my $input_file = shift @ARGV;
@@ -226,6 +247,9 @@ sub split_long {
     foreach my $data_ref (\%composition, \%decomposition, \%stripped) {
       s/\$$data_ref->{name}_DATA/$data_ref->{rawdata}/eg;
     }
+    foreach my $casing (keys %full_casings) {
+      s/\$${casing}CASES_FULL/$full_casings{$casing}/eg;
+    }
     print $output $_;
   }
   close $input;

gen/template/unicode.cpp

@@ -15,6 +15,27 @@
 namespace ufal {
 namespace unilib {
 
+std::u32string unicode::lowercase_full(char32_t chr) {
+  switch (chr) {
+    $LOWERCASES_FULL
+  }
+  return std::u32string(1, unicode::lowercase(chr));
+}
+
+std::u32string unicode::uppercase_full(char32_t chr) {
+  switch (chr) {
+    $UPPERCASES_FULL
+  }
+  return std::u32string(1, unicode::uppercase(chr));
+}
+
+std::u32string unicode::titlecase_full(char32_t chr) {
+  switch (chr) {
+    $TITLECASES_FULL
+  }
+  return std::u32string(1, unicode::titlecase(chr));
+}
+
 const uint8_t unicode::category_index[unicode::CHARS >> 8] = $CATEGORY_INDICES;
 
 const uint8_t unicode::category_block[][256] = $CATEGORY_BLOCKS;

gen/template/unicode.h

@@ -70,6 +70,13 @@ class unicode {
   static inline char32_t uppercase(char32_t chr);
   static inline char32_t titlecase(char32_t chr);
 
+  // Returns the full lowercase/uppercase/titlecase mapping of the given code
+  // point from [Unicode Character Database](http://www.unicode.org/reports/tr44/).
+  // If no full mapping is defined, return the corresponding simple mapping.
+  static std::u32string lowercase_full(char32_t chr);
+  static std::u32string uppercase_full(char32_t chr);
+  static std::u32string titlecase_full(char32_t chr);
+
  private:
   inline static const char32_t CHARS = 0x110000;
   inline static const int32_t DEFAULT_CAT = Cn;

tests/.gitignore

@@ -1,6 +1,7 @@
 .build/
 test_compile
 test_conversion
+test_full_casing
 test_normalization
 test_strip
 test_ucd

tests/Makefile

@@ -11,7 +11,7 @@ include Makefile.builtem
 include ../unilib/Makefile.include
 
 CPP_STANDARD = c++17
-TESTS = $(foreach test,compile conversion normalization strip ucd,test_$(test))
+TESTS = $(foreach test,compile conversion full_casing normalization strip ucd,test_$(test))
 
 .PHONY: all compile test
 all: compile
@@ -21,6 +21,8 @@ test: $(addprefix run_,$(TESTS))
 run_test_compile: $(call exe,test_compile)
 run_test_conversion: $(call exe,test_conversion)
 	python3 test_conversion_input.py | $(call platform_name,./$<)
+run_test_full_casing: $(call exe,test_full_casing)
+	$(call platform_name,./$<) <../gen/data/SpecialCasing.txt
 run_test_normalization: $(call exe,test_normalization)
 	python3 test_normalization_input.py ../gen/data/NormalizationTest.txt | $(call platform_name,./$<)
 run_test_strip: $(call exe,test_strip)
@@ -31,6 +33,7 @@ run_test_ucd: $(call exe,test_ucd)
 C_FLAGS += $(call include_dir,../unilib)
 $(call exe,test_compile): $(call obj,$(addprefix ../unilib/,$(UNILIB_OBJECTS)))
 $(call exe,test_conversion): $(call obj,$(addprefix ../unilib/,unicode utf8 utf16))
+$(call exe,test_full_casing): $(call obj,$(addprefix ../unilib/,unicode))
 $(call exe,test_normalization): $(call obj,$(addprefix ../unilib/,uninorms))
 $(call exe,test_strip): $(call obj,$(addprefix ../unilib/,unicode uninorms unistrip))
 $(call exe,test_ucd): $(call obj,$(addprefix ../unilib/,unicode))

tests/test_full_casing.cpp

@@ -0,0 +1,42 @@
+// This file is part of UniLib <http://github.com/ufal/unilib/>.
+//
+// Copyright 2014 Institute of Formal and Applied Linguistics, Faculty of
+// Mathematics and Physics, Charles University in Prague, Czech Republic.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include "test.h"
+
+#include "unicode.h"
+using namespace ufal::unilib;
+
+int main(void) {
+  vector full_casing_methods = {unicode::lowercase_full, unicode::titlecase_full, unicode::uppercase_full};
+
+  // Load the SpecialCasing file
+  string line;
+  vector<string> columns, codepoints;
+  while (getline(cin, line)) {
+    if (line.empty() || line.compare(0, 1, "#") == 0) continue;
+
+    split(line, ';', columns);
+    if (columns.size() == 6) continue; // Ignore language-specific mappings.
+    if (columns.size() != 5) return cerr << "Cannot parse SpecialCasing line " << line << endl, 1;
+
+    char32_t code = stoi(columns[0], nullptr, 16);
+
+    for (size_t i = 0; i < full_casing_methods.size(); i++) {
+      u32string correct;
+      split(columns[1 + i], ' ', codepoints);
+      for (const auto& codepoint : codepoints)
+        if (!codepoint.empty())
+          correct.push_back(stoi(codepoint, nullptr, 16));
+
+      test(full_casing_methods[i], code, correct);
+    }
+  }
+
+  return test_summary();
+}

unilib/unicode.cpp

@@ -15,6 +15,175 @@
 namespace ufal {
 namespace unilib {
 
+std::u32string unicode::lowercase_full(char32_t chr) {
+  switch (chr) {
+    case U'\u0130': return U"\u0069\u0307";
+  }
+  return std::u32string(1, unicode::lowercase(chr));
+}
+
+std::u32string unicode::uppercase_full(char32_t chr) {
+  switch (chr) {
+    case U'\u00DF': return U"\u0053\u0053";
+    case U'\u0149': return U"\u02BC\u004E";
+    case U'\u01F0': return U"\u004A\u030C";
+    case U'\u0390': return U"\u0399\u0308\u0301";
+    case U'\u03B0': return U"\u03A5\u0308\u0301";
+    case U'\u0587': return U"\u0535\u0552";
+    case U'\u1E96': return U"\u0048\u0331";
+    case U'\u1E97': return U"\u0054\u0308";
+    case U'\u1E98': return U"\u0057\u030A";
+    case U'\u1E99': return U"\u0059\u030A";
+    case U'\u1E9A': return U"\u0041\u02BE";
+    case U'\u1F50': return U"\u03A5\u0313";
+    case U'\u1F52': return U"\u03A5\u0313\u0300";
+    case U'\u1F54': return U"\u03A5\u0313\u0301";
+    case U'\u1F56': return U"\u03A5\u0313\u0342";
+    case U'\u1F80': return U"\u1F08\u0399";
+    case U'\u1F81': return U"\u1F09\u0399";
+    case U'\u1F82': return U"\u1F0A\u0399";
+    case U'\u1F83': return U"\u1F0B\u0399";
+    case U'\u1F84': return U"\u1F0C\u0399";
+    case U'\u1F85': return U"\u1F0D\u0399";
+    case U'\u1F86': return U"\u1F0E\u0399";
+    case U'\u1F87': return U"\u1F0F\u0399";
+    case U'\u1F88': return U"\u1F08\u0399";
+    case U'\u1F89': return U"\u1F09\u0399";
+    case U'\u1F8A': return U"\u1F0A\u0399";
+    case U'\u1F8B': return U"\u1F0B\u0399";
+    case U'\u1F8C': return U"\u1F0C\u0399";
+    case U'\u1F8D': return U"\u1F0D\u0399";
+    case U'\u1F8E': return U"\u1F0E\u0399";
+    case U'\u1F8F': return U"\u1F0F\u0399";
+    case U'\u1F90': return U"\u1F28\u0399";
+    case U'\u1F91': return U"\u1F29\u0399";
+    case U'\u1F92': return U"\u1F2A\u0399";
+    case U'\u1F93': return U"\u1F2B\u0399";
+    case U'\u1F94': return U"\u1F2C\u0399";
+    case U'\u1F95': return U"\u1F2D\u0399";
+    case U'\u1F96': return U"\u1F2E\u0399";
+    case U'\u1F97': return U"\u1F2F\u0399";
+    case U'\u1F98': return U"\u1F28\u0399";
+    case U'\u1F99': return U"\u1F29\u0399";
+    case U'\u1F9A': return U"\u1F2A\u0399";
+    case U'\u1F9B': return U"\u1F2B\u0399";
+    case U'\u1F9C': return U"\u1F2C\u0399";
+    case U'\u1F9D': return U"\u1F2D\u0399";
+    case U'\u1F9E': return U"\u1F2E\u0399";
+    case U'\u1F9F': return U"\u1F2F\u0399";
+    case U'\u1FA0': return U"\u1F68\u0399";
+    case U'\u1FA1': return U"\u1F69\u0399";
+    case U'\u1FA2': return U"\u1F6A\u0399";
+    case U'\u1FA3': return U"\u1F6B\u0399";
+    case U'\u1FA4': return U"\u1F6C\u0399";
+    case U'\u1FA5': return U"\u1F6D\u0399";
+    case U'\u1FA6': return U"\u1F6E\u0399";
+    case U'\u1FA7': return U"\u1F6F\u0399";
+    case U'\u1FA8': return U"\u1F68\u0399";
+    case U'\u1FA9': return U"\u1F69\u0399";
+    case U'\u1FAA': return U"\u1F6A\u0399";
+    case U'\u1FAB': return U"\u1F6B\u0399";
+    case U'\u1FAC': return U"\u1F6C\u0399";
+    case U'\u1FAD': return U"\u1F6D\u0399";
+    case U'\u1FAE': return U"\u1F6E\u0399";
+    case U'\u1FAF': return U"\u1F6F\u0399";
+    case U'\u1FB2': return U"\u1FBA\u0399";
+    case U'\u1FB3': return U"\u0391\u0399";
+    case U'\u1FB4': return U"\u0386\u0399";
+    case U'\u1FB6': return U"\u0391\u0342";
+    case U'\u1FB7': return U"\u0391\u0342\u0399";
+    case U'\u1FBC': return U"\u0391\u0399";
+    case U'\u1FC2': return U"\u1FCA\u0399";
+    case U'\u1FC3': return U"\u0397\u0399";
+    case U'\u1FC4': return U"\u0389\u0399";
+    case U'\u1FC6': return U"\u0397\u0342";
+    case U'\u1FC7': return U"\u0397\u0342\u0399";
+    case U'\u1FCC': return U"\u0397\u0399";
+    case U'\u1FD2': return U"\u0399\u0308\u0300";
+    case U'\u1FD3': return U"\u0399\u0308\u0301";
+    case U'\u1FD6': return U"\u0399\u0342";
+    case U'\u1FD7': return U"\u0399\u0308\u0342";
+    case U'\u1FE2': return U"\u03A5\u0308\u0300";
+    case U'\u1FE3': return U"\u03A5\u0308\u0301";
+    case U'\u1FE4': return U"\u03A1\u0313";
+    case U'\u1FE6': return U"\u03A5\u0342";
+    case U'\u1FE7': return U"\u03A5\u0308\u0342";
+    case U'\u1FF2': return U"\u1FFA\u0399";
+    case U'\u1FF3': return U"\u03A9\u0399";
+    case U'\u1FF4': return U"\u038F\u0399";
+    case U'\u1FF6': return U"\u03A9\u0342";
+    case U'\u1FF7': return U"\u03A9\u0342\u0399";
+    case U'\u1FFC': return U"\u03A9\u0399";
+    case U'\uFB00': return U"\u0046\u0046";
+    case U'\uFB01': return U"\u0046\u0049";
+    case U'\uFB02': return U"\u0046\u004C";
+    case U'\uFB03': return U"\u0046\u0046\u0049";
+    case U'\uFB04': return U"\u0046\u0046\u004C";
+    case U'\uFB05': return U"\u0053\u0054";
+    case U'\uFB06': return U"\u0053\u0054";
+    case U'\uFB13': return U"\u0544\u0546";
+    case U'\uFB14': return U"\u0544\u0535";
+    case U'\uFB15': return U"\u0544\u053B";
+    case U'\uFB16': return U"\u054E\u0546";
+    case U'\uFB17': return U"\u0544\u053D";
+  }
+  return std::u32string(1, unicode::uppercase(chr));
+}
+
+std::u32string unicode::titlecase_full(char32_t chr) {
+  switch (chr) {
+    case U'\u00DF': return U"\u0053\u0073";
+    case U'\u0149': return U"\u02BC\u004E";
+    case U'\u01F0': return U"\u004A\u030C";
+    case U'\u0390': return U"\u0399\u0308\u0301";
+    case U'\u03B0': return U"\u03A5\u0308\u0301";
+    case U'\u0587': return U"\u0535\u0582";
+    case U'\u1E96': return U"\u0048\u0331";
+    case U'\u1E97': return U"\u0054\u0308";
+    case U'\u1E98': return U"\u0057\u030A";
+    case U'\u1E99': return U"\u0059\u030A";
+    case U'\u1E9A': return U"\u0041\u02BE";
+    case U'\u1F50': return U"\u03A5\u0313";
+    case U'\u1F52': return U"\u03A5\u0313\u0300";
+    case U'\u1F54': return U"\u03A5\u0313\u0301";
+    case U'\u1F56': return U"\u03A5\u0313\u0342";
+    case U'\u1FB2': return U"\u1FBA\u0345";
+    case U'\u1FB4': return U"\u0386\u0345";
+    case U'\u1FB6': return U"\u0391\u0342";
+    case U'\u1FB7': return U"\u0391\u0342\u0345";
+    case U'\u1FC2': return U"\u1FCA\u0345";
+    case U'\u1FC4': return U"\u0389\u0345";
+    case U'\u1FC6': return U"\u0397\u0342";
+    case U'\u1FC7': return U"\u0397\u0342\u0345";
+    case U'\u1FD2': return U"\u0399\u0308\u0300";
+    case U'\u1FD3': return U"\u0399\u0308\u0301";
+    case U'\u1FD6': return U"\u0399\u0342";
+    case U'\u1FD7': return U"\u0399\u0308\u0342";
+    case U'\u1FE2': return U"\u03A5\u0308\u0300";
+    case U'\u1FE3': return U"\u03A5\u0308\u0301";
+    case U'\u1FE4': return U"\u03A1\u0313";
+    case U'\u1FE6': return U"\u03A5\u0342";
+    case U'\u1FE7': return U"\u03A5\u0308\u0342";
+    case U'\u1FF2': return U"\u1FFA\u0345";
+    case U'\u1FF4': return U"\u038F\u0345";
+    case U'\u1FF6': return U"\u03A9\u0342";
+    case U'\u1FF7': return U"\u03A9\u0342\u0345";
+    case U'\uFB00': return U"\u0046\u0066";
+    case U'\uFB01': return U"\u0046\u0069";
+    case U'\uFB02': return U"\u0046\u006C";
+    case U'\uFB03': return U"\u0046\u0066\u0069";
+    case U'\uFB04': return U"\u0046\u0066\u006C";
+    case U'\uFB05': return U"\u0053\u0074";
+    case U'\uFB06': return U"\u0053\u0074";
+    case U'\uFB13': return U"\u0544\u0576";
+    case U'\uFB14': return U"\u0544\u0565";
+    case U'\uFB15': return U"\u0544\u056B";
+    case U'\uFB16': return U"\u054E\u0576";
+    case U'\uFB17': return U"\u0544\u056D";
+  }
+  return std::u32string(1, unicode::titlecase(chr));
+}
+
 const uint8_t unicode::category_index[unicode::CHARS >> 8] = {
   0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,17,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,33,41,42,43,44,45,46,47,48,39,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,49,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,50,17,17,17,51,17,52,53,54,55,56,57,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,58,59,59,59,59,59,59,59,59,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,17,61,62,17,63,64,65,66,67,68,69,70,71,17,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,17,17,17,97,98,99,100,100,100,100,100,100,100,100,100,101,17,17,17,17,102,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,17,17,103,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,17,17,104,105,100,100,106,107,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,108,17,17,17,17,109,110,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,111,17,112,113,100,100,100,100,100,100,100,100,100,114,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,115,116,117,118,119,120,121,122,123,39,39,124,100,100,100,100,125,126,127,128,100,129,100,100,130,131,132,100,100,133,134,135,100,136,137,138,139,39,39,140,141,142,39,143,144,100,100,100,100,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,
     17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,145,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,146,147,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,148,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,149,17,17,150,100,100,100,100,100,100,100,100,100,17,17,151,100,100,100,100,100,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,152,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,153,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,

unilib/unicode.h

@@ -70,6 +70,13 @@ class unicode {
   static inline char32_t uppercase(char32_t chr);
   static inline char32_t titlecase(char32_t chr);
 
+  // Returns the full lowercase/uppercase/titlecase mapping of the given code
+  // point from [Unicode Character Database](http://www.unicode.org/reports/tr44/).
+  // If no full mapping is defined, return the corresponding simple mapping.
+  static std::u32string lowercase_full(char32_t chr);
+  static std::u32string uppercase_full(char32_t chr);
+  static std::u32string titlecase_full(char32_t chr);
+
  private:
   inline static const char32_t CHARS = 0x110000;
   inline static const int32_t DEFAULT_CAT = Cn;