Add encodeAsPieces and support commonjs require

.vscode/settings.json

@@ -0,0 +1,16 @@
+{
+    "files.associations": {
+        "*.tcc": "cpp",
+        "cstring": "cpp",
+        "string": "cpp",
+        "string_view": "cpp",
+        "fstream": "cpp",
+        "iomanip": "cpp",
+        "istream": "cpp",
+        "ostream": "cpp",
+        "sstream": "cpp",
+        "stdexcept": "cpp",
+        "system_error": "cpp",
+        "typeinfo": "cpp"
+    }
+}

README.md

@@ -1,3 +1,51 @@
 # Javascript wrapper for the sentencepiece library
 
-Sentencepiece is compiled to webassembly using emscripten.
+## Build
+
+Sentencepiece is compiled to webassembly using emscripten.
+
+To rebuild this project
+
+```bash
+
+yarn
+
+git clone https://github.com/google/sentencepiece.git
+
+yarn build
+
+```
+
+## Use
+
+To use this tool
+
+```js
+
+const { SentencePieceProcessor, cleanText } = require("../dist");
+const ROOT = require('app-root-path')
+
+async function main() {
+
+    let text = "I am still waiting on my card?"
+    let cleaned = cleanText(text)
+
+    let spp = new SentencePieceProcessor()
+    await spp.load(`${ROOT}/test/30k-clean.model`)
+    let ids = spp.encodeIds(cleaned)
+    console.log(ids)
+    let str = spp.decodeIds(ids) // list ids->number
+    console.log(str)
+
+    let pieces = spp.encodePieces(cleaned) // list tokens->string
+    console.log(pieces)
+}
+main()
+
+```
+
+## Note
+
+- devilyouwei updated this repo to make this module support the js `require` keyword and added the using example.
+
+- 2023-1-10, devilyouwei added `encodePieces`.

bindings/sentencepiece.cpp

@@ -2,57 +2,75 @@
 #include <emscripten.h>
 #include "../sentencepiece/src/sentencepiece_processor.h"
 
-class StringView {
-  private: 
-    std::string str;
-    absl::string_view view;
-  public:
-    StringView(const std::string &input) {
-      str = std::string(input);
-      view = absl::string_view(str.data(),str.length());
-    }
-    absl::string_view get_view() {
-      return this->view;
-    }
+using namespace emscripten;
+using namespace std;
+using namespace absl;
+
+class StringView
+{
+private:
+  string str;
+  string_view view;
+
+public:
+  StringView(const string &input)
+  {
+    str = string(input);
+    view = string_view(str.data(), str.length());
+  }
+  string_view get_view()
+  {
+    return this->view;
+  }
 };
 
-template <typename T> emscripten::val vecToView(std::vector<T> vec) {
-  return emscripten::val(emscripten::typed_memory_view(vec.size(),vec.data()));
+/*
+template <typename T>
+val vecToIntArray(vector<T> vec)
+{
+  return val(typed_memory_view(vec.size(), vec.data()));
+}
+
+template <typename T>
+val vecToStringArray(vector<T> vec)
+{
+  return val(vec);
 }
+*/
 
-EMSCRIPTEN_BINDINGS(sentencepiece) {
-  emscripten::register_vector<std::string>("VectorString");
-  emscripten::register_vector<int>("VectorInt");
+EMSCRIPTEN_BINDINGS(sentencepiece)
+{
+  register_vector<string>("VectorString");
+  register_vector<int>("VectorInt");
 
-  emscripten::function("vecToView",emscripten::select_overload<emscripten::val(std::vector<int>)>(&vecToView));
-  emscripten::function("vecFromJSArray",emscripten::select_overload<std::vector<int>(const emscripten::val &)>(&emscripten::vecFromJSArray));
+  /*
+    emscripten::function("vecToIntArray", select_overload<val(vector<int>)>(&vecToIntArray));
+    emscripten::function("vecToStringArray", select_overload<val(vector<string>)>(&vecToStringArray));
+  */
+  emscripten::function("vecFromJSArray", select_overload<vector<int>(const val &)>(&vecFromJSArray));
 
   emscripten::class_<sentencepiece::util::Status>("Status")
-    .constructor()
-    .function("ToString",&sentencepiece::util::Status::ToString)
-    ;
+      .constructor()
+      .function("ToString", &sentencepiece::util::Status::ToString);
 
-  emscripten::class_<absl::string_view>("AbslStringView")
-    .constructor<const std::string &>()
-    ;
+  emscripten::class_<string_view>("AbslStringView")
+      .constructor<const string &>();
 
   emscripten::class_<StringView>("StringView")
-    .constructor<const std::string &>()
-    .function("getView",&StringView::get_view)
-    ;
+      .constructor<const string &>()
+      .function("getView", &StringView::get_view);
 
   emscripten::class_<sentencepiece::SentencePieceProcessor>("SentencePieceProcessor")
-    .smart_ptr_constructor("SentencePieceProcessor", &std::make_shared<sentencepiece::SentencePieceProcessor>)
-    .function("Load", emscripten::select_overload<sentencepiece::util::Status(absl::string_view)>(&sentencepiece::SentencePieceProcessor::Load))
-    .function("status", &sentencepiece::SentencePieceProcessor::status)
-    .function("SetEncodeExtraOptions", &sentencepiece::SentencePieceProcessor::SetEncodeExtraOptions)
-    .function("SetDecodeExtraOptions", &sentencepiece::SentencePieceProcessor::SetDecodeExtraOptions)
-    .function("SetVocabulary", &sentencepiece::SentencePieceProcessor::SetVocabulary)
-    .function("ResetVocabulary", &sentencepiece::SentencePieceProcessor::ResetVocabulary)
-    .function("LoadVocabulary", &sentencepiece::SentencePieceProcessor::LoadVocabulary)
-    .function("EncodeAsPieces", &sentencepiece::SentencePieceProcessor::EncodeAsPieces)
-    .function("EncodeAsIds", &sentencepiece::SentencePieceProcessor::EncodeAsIds)
-    .function("PieceToId", &sentencepiece::SentencePieceProcessor::PieceToId)
-    .function("DecodeIds", &sentencepiece::SentencePieceProcessor::DecodeIds)
-    ;
+      .smart_ptr_constructor("SentencePieceProcessor", &std::make_shared<sentencepiece::SentencePieceProcessor>)
+      .function("Load", emscripten::select_overload<sentencepiece::util::Status(string_view)>(&sentencepiece::SentencePieceProcessor::Load))
+      .function("status", &sentencepiece::SentencePieceProcessor::status)
+      .function("SetEncodeExtraOptions", &sentencepiece::SentencePieceProcessor::SetEncodeExtraOptions)
+      .function("SetDecodeExtraOptions", &sentencepiece::SentencePieceProcessor::SetDecodeExtraOptions)
+      .function("SetVocabulary", &sentencepiece::SentencePieceProcessor::SetVocabulary)
+      .function("ResetVocabulary", &sentencepiece::SentencePieceProcessor::ResetVocabulary)
+      .function("LoadVocabulary", &sentencepiece::SentencePieceProcessor::LoadVocabulary)
+      .function("EncodeAsPieces", &sentencepiece::SentencePieceProcessor::EncodeAsPieces)
+      .function("EncodeAsIds", &sentencepiece::SentencePieceProcessor::EncodeAsIds)
+      .function("PieceToId", &sentencepiece::SentencePieceProcessor::PieceToId)
+      .function("DecodeIds", &sentencepiece::SentencePieceProcessor::DecodeIds);
 }

package.json

@@ -1,9 +1,8 @@
 {
-  "name": "@weblab-notebook/sentencepiece",
-  "version": "0.1.3",
-  "description": "Sentencepiece tokenization for natural language processing.",
+  "name": "sentencepiece-js",
+  "version": "1.1.0",
+  "description": "Sentencepiece tokenization for natural language processing, JS version.",
   "main": "dist/index.js",
-  "type": "module",
   "exports": {
     "imports": "./dist/index.js",
     "default": "./dist/index.js"
@@ -24,14 +23,15 @@
   "keywords": [
     "machine_learning",
     "albert",
-    "nlp"
+    "nlp",
+    "sentencepiece"
   ],
-  "author": "Jan Kaul",
+  "author": "devilyouwei, Jan Kaul",
   "license": "Apache-2.0",
   "bugs": {
     "url": "https://github.com/JanKaul/sentencepiece/issues"
   },
-  "homepage": "https://github.com/JanKaul/sentencepiece#readme",
+  "homepage": "https://github.com/devilyouwei/sentencepiece",
   "devDependencies": {
     "@esm-bundle/chai": "^4.3.4-fix.0",
     "@rollup/plugin-commonjs": "^20.0.0",
@@ -43,5 +43,8 @@
     "rollup": "^2.56.2",
     "tslib": "^2.3.1",
     "typescript": "^4.4.3"
+  },
+  "dependencies": {
+    "app-root-path": "^3.1.0"
   }
 }

rollup.config.js

@@ -7,7 +7,7 @@ export default {
     input: 'src/index.ts',
     output: {
         dir: 'dist',
-        format: 'es'
+        format: 'cjs'
     },
-    plugins: [nodeResolve({ browser: true }), commonjs(), wasm(), typescript({ target: "es2017", downlevelIteration: true })]
+    plugins: [nodeResolve({ browser: true }), commonjs(), wasm(), typescript({ target: "es5", downlevelIteration: true })]
 };

src/index.ts

@@ -1,4 +1,4 @@
-import { sentencePieceProcessor, cleanText } from "./sentencePieceProcessor";
+import { SentencePieceProcessor, cleanText } from "./sentencePieceProcessor";
 
-export { sentencePieceProcessor, cleanText }
-export default { sentencePieceProcessor, cleanText }
+export { SentencePieceProcessor, cleanText }
+export default { SentencePieceProcessor, cleanText }