From 862451acb836bd08c3e6e69b5454c085cdd7a07e Mon Sep 17 00:00:00 2001 From: "Alexandre E. Souza" Date: Thu, 12 Sep 2024 21:42:38 -0300 Subject: [PATCH] llms: Add fake package (#935) * feat: add fake package --- .../models/llms/Integrations/fake.mdx | 88 +++++++++++ llms/fake/fakellm.go | 57 +++++++ llms/fake/fakellm_test.go | 144 ++++++++++++++++++ 3 files changed, 289 insertions(+) create mode 100644 docs/docs/modules/model_io/models/llms/Integrations/fake.mdx create mode 100644 llms/fake/fakellm.go create mode 100644 llms/fake/fakellm_test.go diff --git a/docs/docs/modules/model_io/models/llms/Integrations/fake.mdx b/docs/docs/modules/model_io/models/llms/Integrations/fake.mdx new file mode 100644 index 000000000..6e9781efb --- /dev/null +++ b/docs/docs/modules/model_io/models/llms/Integrations/fake.mdx @@ -0,0 +1,88 @@ +--- +sidebar_label: Fake LLM +--- + +# Fake LLM + +## Overview + +This documentation provides an overview of the `fake` package, which offers a simulated implementation of a Language Learning Model (LLM) for testing purposes in Go applications. + +## Installation + +To use the `fake` package, import it into your Go project: + +```bash +go get "github.com/tmc/langchaingo" +``` + + + +## Prerequisites +Ensure you have Go programming language installed on your machine (version 1.15 or higher recommended). + +## Example Usage +Here is an example demonstrating how to use the fake package: + + +```go +package main + +import ( + "context" + "fmt" + "log" + + "github.com/tmc/langchaingo/llms" + "github.com/tmc/langchaingo/llms/fake" +) + +func main() { + // Creating a fake LLM with initial responses. + responses := []string{ + "Hello!", + "How are you?", + "I'm fine, thanks.", + } + llm := fake.NewFakeLLM(responses) + + // Calling the fake LLM with a prompt. + ctx := context.Background() + response, err := llm.Call(ctx, "Hi there!") + if err != nil { + fmt.Printf("Error calling LLM: %v\n", err) + } else { + fmt.Println("LLM Response:", response) + } + + // Adding a new response and testing again. + llm.AddResponse("Goodbye!") + response, err = llm.Call(ctx, "See you later!") + if err != nil { + fmt.Printf("Error calling LLM: %v\n", err) + } else { + fmt.Println("LLM Response:", response) + } +} +``` + +# API Reference +`NewFakeLLM(responses []string) *LLM` + +Creates a new instance of the fake LLM with the provided responses. + +`LLM.Call(ctx context.Context, prompt string) (string, error)` + +Simulates calling the model with a specific prompt and returns a fictional response. + +`LLM.Reset()` + +Resets the fake LLM, allowing responses to cycle through again. + +`LLM.AddResponse(response string)` + +Adds a new response to the list of possible responses of the fake LLM. + +# Purpose + +The fake package is designed to facilitate testing of applications that interact with language learning models, without relying on real model implementations. It helps validate application logic and behavior in a controlled environment. \ No newline at end of file diff --git a/llms/fake/fakellm.go b/llms/fake/fakellm.go new file mode 100644 index 000000000..146d10cb9 --- /dev/null +++ b/llms/fake/fakellm.go @@ -0,0 +1,57 @@ +package fake + +import ( + "context" + "errors" + + "github.com/tmc/langchaingo/llms" +) + +type LLM struct { + responses []string + index int +} + +func NewFakeLLM(responses []string) *LLM { + return &LLM{ + responses: responses, + index: 0, + } +} + +// GenerateContent generate fake content. +func (f *LLM) GenerateContent(_ context.Context, _ []llms.MessageContent, _ ...llms.CallOption) (*llms.ContentResponse, error) { + if len(f.responses) == 0 { + return nil, errors.New("no responses configured") + } + if f.index >= len(f.responses) { + f.index = 0 // reset index + } + response := f.responses[f.index] + f.index++ + return &llms.ContentResponse{ + Choices: []*llms.ContentChoice{{Content: response}}, + }, nil +} + +// Call the model with a prompt. +func (f *LLM) Call(ctx context.Context, prompt string, options ...llms.CallOption) (string, error) { + resp, err := f.GenerateContent(ctx, []llms.MessageContent{{Role: llms.ChatMessageTypeHuman, Parts: []llms.ContentPart{llms.TextContent{Text: prompt}}}}, options...) + if err != nil { + return "", err + } + if len(resp.Choices) < 1 { + return "", errors.New("empty response from model") + } + return resp.Choices[0].Content, nil +} + +// Reset the index to 0. +func (f *LLM) Reset() { + f.index = 0 +} + +// AddResponse adds a response to the list of responses. +func (f *LLM) AddResponse(response string) { + f.responses = append(f.responses, response) +} diff --git a/llms/fake/fakellm_test.go b/llms/fake/fakellm_test.go new file mode 100644 index 000000000..7f39bd7a3 --- /dev/null +++ b/llms/fake/fakellm_test.go @@ -0,0 +1,144 @@ +package fake + +import ( + "context" + "testing" + + "github.com/tmc/langchaingo/chains" + "github.com/tmc/langchaingo/llms" + "github.com/tmc/langchaingo/memory" +) + +func TestFakeLLM_CallMethod(t *testing.T) { + t.Parallel() + responses := setupResponses() + fakeLLM := NewFakeLLM(responses) + ctx := context.Background() + + if output, _ := fakeLLM.Call(ctx, "Teste"); output != responses[0] { + t.Errorf("Expected 'Resposta 1', got '%s'", output) + } + + if output, _ := fakeLLM.Call(ctx, "Teste"); output != responses[1] { + t.Errorf("Expected 'Resposta 2', got '%s'", output) + } + + if output, _ := fakeLLM.Call(ctx, "Teste"); output != responses[2] { + t.Errorf("Expected 'Resposta 3', got '%s'", output) + } + + // Testa reinicialização automática + if output, _ := fakeLLM.Call(ctx, "Teste"); output != responses[0] { + t.Errorf("Expected 'Resposta 1', got '%s'", output) + } +} + +func TestFakeLLM_GenerateContentMethod(t *testing.T) { + t.Parallel() + responses := setupResponses() + fakeLLM := NewFakeLLM(responses) + ctx := context.Background() + msg := llms.MessageContent{ + Role: llms.ChatMessageTypeHuman, + Parts: []llms.ContentPart{llms.TextContent{Text: "Teste"}}, + } + + resp, err := fakeLLM.GenerateContent(ctx, []llms.MessageContent{msg}) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if len(resp.Choices) < 1 || resp.Choices[0].Content != responses[0] { + t.Errorf("Expected 'Resposta 1', got '%s'", resp.Choices[0].Content) + } + + resp, err = fakeLLM.GenerateContent(ctx, []llms.MessageContent{msg}) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if len(resp.Choices) < 1 || resp.Choices[0].Content != responses[1] { + t.Errorf("Expected 'Resposta 2', got '%s'", resp.Choices[0].Content) + } + + resp, err = fakeLLM.GenerateContent(ctx, []llms.MessageContent{msg}) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if len(resp.Choices) < 1 || resp.Choices[0].Content != responses[2] { + t.Errorf("Expected 'Resposta 1', got '%s'", resp.Choices[0].Content) + } +} + +func TestFakeLLM_ResetMethod(t *testing.T) { + t.Parallel() + responses := setupResponses() + fakeLLM := NewFakeLLM(responses) + ctx := context.Background() + + fakeLLM.Reset() + if output, _ := fakeLLM.Call(ctx, "Teste"); output != responses[0] { + t.Errorf("Expected 'Resposta 1', got '%s'", output) + } +} + +func TestFakeLLM_AddResponseMethod(t *testing.T) { + t.Parallel() + responses := setupResponses() + fakeLLM := NewFakeLLM(responses) + ctx := context.Background() + + fakeLLM.AddResponse("Resposta 4") + fakeLLM.Reset() + _, err := fakeLLM.Call(ctx, "Teste") + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + _, err = fakeLLM.Call(ctx, "Teste") + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + _, err = fakeLLM.Call(ctx, "Teste") + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + if output, _ := fakeLLM.Call(ctx, "Teste"); output != "Resposta 4" { + t.Errorf("Expected 'Resposta 4', got '%s'", output) + } +} + +func TestFakeLLM_WithChain(t *testing.T) { + t.Parallel() + responses := setupResponses() + fakeLLM := NewFakeLLM(responses) + ctx := context.Background() + + fakeLLM.AddResponse("My name is Alexandre") + + NextToResponse(fakeLLM, 4) + llmChain := chains.NewConversation(fakeLLM, memory.NewConversationBuffer()) + out, err := chains.Run(ctx, llmChain, "What's my name? How many times did I ask this?") + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + if out != "My name is Alexandre" { + t.Errorf("Expected 'My name is Alexandre', got '%s'", out) + } +} + +func setupResponses() []string { + return []string{ + "Resposta 1", + "Resposta 2", + "Resposta 3", + } +} + +func NextToResponse(fakeLLM *LLM, n int) { + for i := 1; i < n; i++ { + _, err := fakeLLM.Call(context.Background(), "Teste") + if err != nil { + panic(err) + } + } +}