diff --git a/include/whisper.h b/include/whisper.h index 71949bdd397..8999c1c3636 100644 --- a/include/whisper.h +++ b/include/whisper.h @@ -408,6 +408,9 @@ extern "C" { WHISPER_API const char * whisper_token_to_str(struct whisper_context * ctx, whisper_token token); WHISPER_API const char * whisper_model_type_readable(struct whisper_context * ctx); + // String -> Token Id. Uses the vocabulary in the provided context + WHISPER_API bool whisper_token_exists(struct whisper_context * ctx, const char * str); + WHISPER_API whisper_token whisper_str_to_token(struct whisper_context * ctx, const char * str); // Special tokens WHISPER_API whisper_token whisper_token_eot (struct whisper_context * ctx); diff --git a/src/whisper.cpp b/src/whisper.cpp index bcc530ae891..aea3f20bc71 100644 --- a/src/whisper.cpp +++ b/src/whisper.cpp @@ -4068,6 +4068,14 @@ const char * whisper_token_to_str(struct whisper_context * ctx, whisper_token to return ctx->vocab.id_to_token.at(token).c_str(); } +whisper_token whisper_str_to_token(struct whisper_context * ctx, const char * str) { + return ctx->vocab.token_to_id.at(str); +} + +bool whisper_token_exists(struct whisper_context * ctx, const char * str) { + return ctx->vocab.token_to_id.find(str) != ctx->vocab.token_to_id.end(); +} + whisper_token whisper_token_eot(struct whisper_context * ctx) { return ctx->vocab.token_eot; }