Tokenize Detokenize compatibility

examples/tokenization/tokenization.ts

@@ -0,0 +1,21 @@
+import ollama from '../../src/browser.js'
+
+async function main() {
+  // Tokenize some text
+  const tokResponse = await ollama.tokenize({
+    model: 'llama3.2',
+    text: 'Why is the sky blue?'
+  })
+
+  console.log('Tokens from model:', tokResponse.tokens)
+
+  // Detokenize the tokens back to text
+  const detokResponse = await ollama.detokenize({
+    model: 'llama3.2', 
+    tokens: tokResponse.tokens
+  })
+
+  console.log('Text from model:', detokResponse.text)
+}
+
+main().catch(console.error)

src/browser.ts

@@ -9,6 +9,8 @@
   CopyRequest,
   CreateRequest,
   DeleteRequest,
+  DetokenizeRequest,
+  DetokenizeResponse,
   EmbedRequest,
   EmbedResponse,
   EmbeddingsRequest,
@@ -24,6 +26,8 @@
   ShowRequest,
   ShowResponse,
   StatusResponse,
+  TokenizeRequest,
+  TokenizeResponse,
 } from './interfaces.js'
 
 export class Ollama {
@@ -65,7 +69,7 @@
    */
   protected async processStreamableRequest<T extends object>(
     endpoint: string,
     request: { stream?: boolean } & Record<string, any>,
   ): Promise<T | AbortableAsyncIterator<T>> {
     request.stream = request.stream ?? false
     const host = `${this.config.host}/api/${endpoint}`
@@ -322,9 +326,37 @@
     })
     return (await response.json()) as ListResponse
   }
-}
 
+  /**
+   * Tokenizes text into tokens.
+   * @param request {TokenizeRequest} - The request object.
+   * @returns {Promise<TokenizeResponse>} - The response object.
+   */
+  async tokenize(request: TokenizeRequest): Promise<TokenizeResponse> {
+    const response = await utils.post(this.fetch, `${this.config.host}/api/tokenize`, {
+      ...request,
+    }, {
+      headers: this.config.headers
+    })
+    return (await response.json()) as TokenizeResponse
+  }
+
+  /**
+   * Detokenizes tokens back into text.
+   * @param request {DetokenizeRequest} - The request object.
+   * @returns {Promise<DetokenizeResponse>} - The response object.
+   */
+  async detokenize(request: DetokenizeRequest): Promise<DetokenizeResponse> {
+    const response = await utils.post(this.fetch, `${this.config.host}/api/detokenize`, {
+      ...request,
+    }, {
+      headers: this.config.headers
+    })
+    return (await response.json()) as DetokenizeResponse
+  }
+}
 export default new Ollama()
 
 // export all types from the main entry point so that packages importing types dont need to specify paths
 export * from './interfaces.js'
+

src/interfaces.ts

@@ -71,7 +71,7 @@
   function: {
     name: string;
     arguments: {
       [key: string]: any;
     };
   };
 }
@@ -159,6 +159,16 @@
   options?: Partial<Options>
 }
 
+export interface TokenizeRequest {
+  model: string
+  text: string
+}
+
+export interface DetokenizeRequest {
+  model: string
+  tokens: Int32Array
+}
+
 // response types
 
 export interface GenerateResponse {
@@ -234,14 +244,22 @@
   details: ModelDetails
   messages: Message[]
   modified_at: Date
   model_info: Map<string, any>
   projector_info?: Map<string, any>
 }
 
 export interface ListResponse {
   models: ModelResponse[]
 }
 
+export interface TokenizeResponse {
+  tokens: Int32Array
+}
+
+export interface DetokenizeResponse {
+  text: string
+}
+
 export interface ErrorResponse {
   error: string
 }