Fix the decoding issues

CMakeLists.txt

@@ -712,6 +712,7 @@ add_library(${TARGET}
     ${GGML_SOURCES_SYCL}  ${GGML_HEADERS_SYCL}
     ${GGML_SOURCES_ROCM}  ${GGML_HEADERS_ROCM}
     ${GGML_SOURCES_BLAS}  ${GGML_HEADERS_BLAS}
+    unicode.h
     whisper.h
     whisper.cpp
     )

Makefile

@@ -415,7 +415,7 @@ ggml-blas.o: ggml-blas.cpp ggml-blas.h
 
 WHISPER_OBJ += ggml.o ggml-alloc.o ggml-backend.o ggml-quants.o
 
-whisper.o: whisper.cpp whisper.h whisper-mel.hpp ggml.h ggml-cuda.h
+whisper.o: whisper.cpp whisper.h unicode.h whisper-mel.hpp ggml.h ggml-cuda.h
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
 ifndef WHISPER_COREML
@@ -476,9 +476,10 @@ CC_SDL=`sdl2-config --cflags --libs`
 
 SRC_COMMON     = examples/common.cpp examples/common-ggml.cpp examples/grammar-parser.cpp
 SRC_COMMON_SDL = examples/common-sdl.cpp
+SRC_CONSOLE    = examples/console.cpp
 
 main: examples/main/main.cpp $(SRC_COMMON) $(WHISPER_OBJ)
-	$(CXX) $(CXXFLAGS) examples/main/main.cpp $(SRC_COMMON) $(WHISPER_OBJ) -o main $(LDFLAGS)
+	$(CXX) $(CXXFLAGS) examples/main/main.cpp $(SRC_COMMON) $(SRC_CONSOLE) $(WHISPER_OBJ) -o main $(LDFLAGS)
 	./main -h
 
 bench: examples/bench/bench.cpp $(WHISPER_OBJ)
@@ -491,7 +492,7 @@ server: examples/server/server.cpp $(SRC_COMMON) $(WHISPER_OBJ)
 	$(CXX) $(CXXFLAGS) examples/server/server.cpp $(SRC_COMMON) $(WHISPER_OBJ) -o server $(LDFLAGS) $(LWINSOCK2)
 
 stream: examples/stream/stream.cpp $(SRC_COMMON) $(SRC_COMMON_SDL) $(WHISPER_OBJ)
-	$(CXX) $(CXXFLAGS) examples/stream/stream.cpp $(SRC_COMMON) $(SRC_COMMON_SDL) $(WHISPER_OBJ) -o stream $(CC_SDL) $(LDFLAGS)
+	$(CXX) $(CXXFLAGS) examples/stream/stream.cpp $(SRC_COMMON) $(SRC_CONSOLE) $(SRC_COMMON_SDL) $(WHISPER_OBJ) -o stream $(CC_SDL) $(LDFLAGS)
 
 command: examples/command/command.cpp $(SRC_COMMON) $(SRC_COMMON_SDL) $(WHISPER_OBJ)
 	$(CXX) $(CXXFLAGS) examples/command/command.cpp $(SRC_COMMON) $(SRC_COMMON_SDL) $(WHISPER_OBJ) -o command $(CC_SDL) $(LDFLAGS)
@@ -500,7 +501,7 @@ lsp: examples/lsp/lsp.cpp $(SRC_COMMON) $(SRC_COMMON_SDL) $(WHISPER_OBJ)
 	$(CXX) $(CXXFLAGS) examples/lsp/lsp.cpp $(SRC_COMMON) $(SRC_COMMON_SDL) $(WHISPER_OBJ) -o lsp $(CC_SDL) $(LDFLAGS)
 
 talk: examples/talk/talk.cpp examples/talk/gpt-2.cpp $(SRC_COMMON) $(SRC_COMMON_SDL) $(WHISPER_OBJ)
-	$(CXX) $(CXXFLAGS) examples/talk/talk.cpp examples/talk/gpt-2.cpp $(SRC_COMMON) $(SRC_COMMON_SDL) $(WHISPER_OBJ) -o talk $(CC_SDL) $(LDFLAGS)
+	$(CXX) $(CXXFLAGS) examples/talk/talk.cpp examples/talk/gpt-2.cpp $(SRC_COMMON) $(SRC_CONSOLE) $(SRC_COMMON_SDL) $(WHISPER_OBJ) -o talk $(CC_SDL) $(LDFLAGS)
 
 talk-llama: examples/talk-llama/talk-llama.cpp examples/talk-llama/llama.cpp examples/talk-llama/unicode.cpp examples/talk-llama/unicode-data.cpp $(SRC_COMMON) $(SRC_COMMON_SDL) $(WHISPER_OBJ)
 	$(CXX) $(CXXFLAGS) examples/talk-llama/talk-llama.cpp examples/talk-llama/llama.cpp examples/talk-llama/unicode.cpp examples/talk-llama/unicode-data.cpp $(SRC_COMMON) $(SRC_COMMON_SDL) $(WHISPER_OBJ) -o talk-llama $(CC_SDL) $(LDFLAGS)

bindings/go/params.go

@@ -19,10 +19,6 @@ func (p *Params) SetTranslate(v bool) {
 	p.translate = toBool(v)
 }
 
-func (p *Params) SetSplitOnWord(v bool) {
-	p.split_on_word = toBool(v)
-}
-
 func (p *Params) SetNoContext(v bool) {
 	p.no_context = toBool(v)
 }

bindings/javascript/package.json

@@ -1,26 +1,26 @@
-{
-  "name": "whisper.cpp",
-  "version": "1.6.2",
-  "description": "Whisper speech recognition",
-  "main": "whisper.js",
-  "scripts": {
-    "test": "echo \"todo: add tests\" && exit 0"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/ggerganov/whisper.cpp"
-  },
-  "keywords": [
-    "openai",
-    "whisper",
-    "speech-to-text",
-    "speech-recognition",
-    "transformer"
-  ],
-  "author": "Georgi Gerganov",
-  "license": "MIT",
-  "bugs": {
-    "url": "https://github.com/ggerganov/whisper.cpp/issues"
-  },
-  "homepage": "https://github.com/ggerganov/whisper.cpp#readme"
-}
+{
+  "name": "whisper.cpp",
+  "version": "1.6.2",
+  "description": "Whisper speech recognition",
+  "main": "whisper.js",
+  "scripts": {
+    "test": "echo \"todo: add tests\" && exit 0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ggerganov/whisper.cpp"
+  },
+  "keywords": [
+    "openai",
+    "whisper",
+    "speech-to-text",
+    "speech-recognition",
+    "transformer"
+  ],
+  "author": "Georgi Gerganov",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/ggerganov/whisper.cpp/issues"
+  },
+  "homepage": "https://github.com/ggerganov/whisper.cpp#readme"
+}

examples/CMakeLists.txt

@@ -27,6 +27,8 @@ if (WHISPER_FFMPEG)
 endif()
 
 add_library(${TARGET} STATIC
+    console.h
+    console.cpp
     common.h
     common.cpp
     common-ggml.h

examples/command/command.cpp

@@ -8,6 +8,7 @@
 
 #include "common-sdl.h"
 #include "common.h"
+#include "console.h"
 #include "whisper.h"
 #include "grammar-parser.h"
 
@@ -57,9 +58,9 @@ struct whisper_params {
     std::string suppress_regex;
 };
 
-void whisper_print_usage(int argc, char ** argv, const whisper_params & params);
+void whisper_print_usage(int argc, const char ** argv, const whisper_params & params);
 
-bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
+bool whisper_params_parse(int argc, const char ** argv, whisper_params & params) {
     for (int i = 1; i < argc; i++) {
         std::string arg = argv[i];
 
@@ -99,7 +100,7 @@ bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {
     return true;
 }
 
-void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {
+void whisper_print_usage(int /*argc*/, const char ** argv, const whisper_params & params) {
     fprintf(stderr, "\n");
     fprintf(stderr, "usage: %s [options]\n", argv[0]);
     fprintf(stderr, "\n");
@@ -678,7 +679,7 @@ int process_general_transcription(struct whisper_context * ctx, audio_async & au
     return 0;
 }
 
-int main(int argc, char ** argv) {
+int run(int argc, const char ** argv) {
     whisper_params params;
 
     if (whisper_params_parse(argc, argv, params) == false) {
@@ -775,3 +776,23 @@ int main(int argc, char ** argv) {
 
     return ret_val;
 }
+
+#if _WIN32
+int wmain(int argc, const wchar_t ** argv_UTF16LE) {
+    console::init(true, true);
+    atexit([]() { console::cleanup(); });
+    std::vector<std::string> buffer(argc);
+    std::vector<const char*> argv_UTF8(argc);
+    for (int i = 0; i < argc; ++i) {
+        buffer[i] = console::UTF16toUTF8(argv_UTF16LE[i]);
+        argv_UTF8[i] = buffer[i].c_str();
+    }
+    return run(argc, argv_UTF8.data());
+}
+#else
+int main(int argc, const char ** argv_UTF8) {
+    console::init(true, true);
+    atexit([]() { console::cleanup(); });
+    return run(argc, argv_UTF8);
+}
+#endif

examples/common.cpp

@@ -1,6 +1,7 @@
 #define _USE_MATH_DEFINES // for M_PI
 
 #include "common.h"
+#include "console.h"
 
 // third-party utilities
 // use your favorite implementations
@@ -674,7 +675,11 @@ bool read_wav(const std::string & fname, std::vector<float>& pcmf32, std::vector
             return false;
         }
     }
+#if _WIN32
+    else if (drwav_init_file_w(&wav, console::UTF8toUTF16(fname).c_str(), nullptr) == false) {
+#else
     else if (drwav_init_file(&wav, fname.c_str(), nullptr) == false) {
+#endif
 #if defined(WHISPER_FFMPEG)
         if (ffmpeg_decode_audio(fname, wav_data) != 0) {
             fprintf(stderr, "error: failed to ffmpeg decode '%s' \n", fname.c_str());
@@ -889,7 +894,11 @@ int timestamp_to_sample(int64_t t, int n_samples, int whisper_sample_rate) {
 
 bool is_file_exist(const char *fileName)
 {
-    std::ifstream infile(fileName);
+    #ifdef _WIN32
+        std::wifstream infile(console::UTF8toUTF16(fileName).c_str());
+    #else
+        std::ifstream infile(fileName);
+    #endif
     return infile.good();
 }