diff --git a/README.md b/README.md index 12c7ab0d..83d45cbc 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ As Spezi LLM contains a variety of different targets for specific LLM functional Spezi LLM provides a number of targets to help developers integrate LLMs in their Spezi-based applications: - [SpeziLLM](https://swiftpackageindex.com/stanfordspezi/spezillm/documentation/spezillm): Base infrastructure of LLM execution in the Spezi ecosystem. -- [SpeziLLMLocal](https://swiftpackageindex.com/stanfordspezi/spezillm/documentation/spezillmlocal): Local LLM execution capabilities directly on-device. Enables running open-source LLMs like [Meta's Llama2 models](https://ai.meta.com/llama/). +- [SpeziLLMLocal](https://swiftpackageindex.com/stanfordspezi/spezillm/documentation/spezillmlocal): Local LLM execution capabilities directly on-device. Enables running open-source LLMs from Hugging Face like [Meta's Llama2](https://ai.meta.com/llama/), [Microsoft's Phi](https://azure.microsoft.com/en-us/products/phi), [Google's Gemma](https://ai.google.dev/gemma), or [DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1), among others. See [LLMLocalModel](https://swiftpackageindex.com/stanfordspezi/spezillm/main/documentation/spezillmlocal/llmlocalmodel) for a list of models tested with SpeziLLM. - [SpeziLLMLocalDownload](https://swiftpackageindex.com/stanfordspezi/spezillm/documentation/spezillmlocaldownload): Download and storage manager of local Language Models, including onboarding views. - [SpeziLLMOpenAI](https://swiftpackageindex.com/stanfordspezi/spezillm/documentation/spezillmopenai): Integration with OpenAI's GPT models via using OpenAI's API service. - [SpeziLLMFog](https://swiftpackageindex.com/stanfordspezi/spezillm/documentation/spezillmfog): Discover and dispatch LLM inference jobs to Fog node resources within the local network. @@ -82,6 +82,25 @@ class TestAppDelegate: SpeziAppDelegate { } ``` +[SpeziLLMLocalDownload](https://swiftpackageindex.com/stanfordspezi/spezillm/documentation/spezillmlocaldownload) can be used to download an LLM from [HuggingFace](https://huggingface.co/) and save it on the device for execution. The `LLMLocalDownloadView` provides an out-of-the-box onboarding view for downloading models locally. + +```swift +struct LLMLocalOnboardingDownloadView: View { + var body: some View { + LLMLocalDownloadView( + model: .llama3_8B_4bit, + downloadDescription: "The Llama3 8B model will be downloaded", + ) { + // Action to perform after the model is downloaded and the user presses the next button. + } + } +} +``` + +> [!TIP] +> The `LLMLocalDownloadView` view can be included in your onboarding process using SpeziOnboarding as [demonstrated in this example](https://swiftpackageindex.com/stanfordspezi/spezillm/main/documentation/spezillmlocaldownload/llmlocaldownloadview#overview). + + #### Usage The code example below showcases the interaction with local LLMs through the the [SpeziLLM](https://swiftpackageindex.com/stanfordspezi/spezillm/documentation/spezillm) [`LLMRunner`](https://swiftpackageindex.com/stanfordspezi/spezillm/documentation/spezillm/llmrunner), which is injected into the SwiftUI `Environment` via the `Configuration` shown above. @@ -100,7 +119,6 @@ struct LLMLocalDemoView: View { let llmSession: LLMLocalSession = runner( with: LLMLocalSchema( model: .llama3_8B_4bit, - formatChat: LLMLocalSchema.PromptFormattingDefaults.llama3 ) ) @@ -116,8 +134,22 @@ struct LLMLocalDemoView: View { } ``` +The [`LLMChatViewSchema`](https://swiftpackageindex.com/stanfordspezi/spezillm/main/documentation/spezillm/llmchatviewschema) can be used to easily create a conversational chat interface for your chatbot application with a local LLM. + +```swift +struct LLMLocalChatView: View { + var body: some View { + LLMChatViewSchema( + with: LLMLocalSchema( + model: .llama3_8B_4bit + ) + ) + } +} +``` + > [!NOTE] -> To learn more about the usage of SpeziLLMLocal, please refer to the [DocC documentation](https://swiftpackageindex.com/stanfordspezi/spezillm/documentation/spezillmlocal). +> To learn more about the usage of SpeziLLMLocal, please refer to the comprehensive [DocC documentation](https://swiftpackageindex.com/stanfordspezi/spezillm/documentation/spezillmlocal). ### Spezi LLM Open AI @@ -171,7 +203,7 @@ struct LLMOpenAIDemoView: View { let llmSession: LLMOpenAISession = runner( with: LLMOpenAISchema( parameters: .init( - modelType: .gpt3_5Turbo, + modelType: .gpt4_o, systemPrompt: "You're a helpful assistant that answers questions from users.", overwritingToken: "abc123" ) diff --git a/Sources/SpeziLLMLocal/Configuration/LLMLocalModel.swift b/Sources/SpeziLLMLocal/Configuration/LLMLocalModel.swift index 0e6f9bcf..8dc0aab3 100644 --- a/Sources/SpeziLLMLocal/Configuration/LLMLocalModel.swift +++ b/Sources/SpeziLLMLocal/Configuration/LLMLocalModel.swift @@ -18,6 +18,10 @@ public enum LLMLocalModel { case llama3_2_1B_4bit /// Llama 3.2, 3 Billion Parameters, Instruction-Tuned, 4-bit Version case llama3_2_3B_4bit + /// Llama 3.1 Aloe, Beta 8 Billion Parameters, 4-bit Version + case llama3_1_aloe_8B_4bit + /// Llama 3.0 Med42, 8 Billion Parameters, 4-bit Version + case llama3_med42_8B_4bit /// Mistral Nemo, Instruction-Tuned, Model 2407, 4-bit Version case mistralNeMo4bit /// SmolLM, 135 Million Parameters, Instruction-Tuned, 4-bit Version @@ -39,9 +43,17 @@ public enum LLMLocalModel { /// Gemma 2, 2 Billion Parameters, Instruction-Tuned, 4-bit Version case gemma_2_2b_it_4bit /// Qwen 1.5, 0.5 Billion Parameters, Chat-Tuned, 4-bit Version - case qwen205b4bit + case qwen1_5_0_5b_4bit + /// Qwen 2, 7 Billion Parameters, 4-bit version + case qwen2_7b_4bit /// OpenELM, 270 Million Parameters, Instruction-Tuned case openelm270m4bit + /// DeepSeek R1 Distill Qwen 1.5 Billion Parameters, 8-bit Version + case deepseek_r1_distill_qwen_1_5b_8bit + /// DeepSeek R1 Distill Qwen 7 Billion Parameters, 4-bit Version + case deepseek_r1_distill_qwen_7b_4bit + /// DeepSeek R1 Distill Llama 8 Billion Parameters, 4-bit Version + case deepseek_r1_distill_llama_8b_4bit /// Set the Huggingface ID of the model. e.g. "\/\" case custom(id: String) @@ -56,6 +68,10 @@ public enum LLMLocalModel { return "mlx-community/Llama-3.2-1B-Instruct-4bit" case .llama3_2_3B_4bit: return "mlx-community/Llama-3.2-3B-Instruct-4bit" + case .llama3_1_aloe_8B_4bit: + return "mlx-community/Llama3.1-Aloe-Beta-8B" + case .llama3_med42_8B_4bit: + return "mlx-community/Llama3-Med42-8B" case .mistralNeMo4bit: return "mlx-community/Mistral-Nemo-Instruct-2407-4bit" case .smolLM_135M_4bit: @@ -76,10 +92,18 @@ public enum LLMLocalModel { return "mlx-community/gemma-2-9b-it-4bit" case .gemma_2_2b_it_4bit: return "mlx-community/gemma-2-2b-it-4bit" - case .qwen205b4bit: + case .qwen1_5_0_5b_4bit: return "mlx-community/Qwen1.5-0.5B-Chat-4bit" + case .qwen2_7b_4bit: + return "mlx-community/Qwen2-7B-4bit" case .openelm270m4bit: return "mlx-community/OpenELM-270M-Instruct" + case .deepseek_r1_distill_qwen_1_5b_8bit: + return "mlx-community/DeepSeek-R1-Distill-Qwen-1.5B-8bit" + case .deepseek_r1_distill_qwen_7b_4bit: + return "mlx-community/DeepSeek-R1-Distill-Qwen-7B-4bit" + case .deepseek_r1_distill_llama_8b_4bit: + return "mlx-community/DeepSeek-R1-Distill-Llama-8B-4bit-mlx" case .custom(let id): return id } diff --git a/Sources/SpeziLLMLocalDownload/LLMLocalDownloadView.swift b/Sources/SpeziLLMLocalDownload/LLMLocalDownloadView.swift index bacf29db..43240159 100644 --- a/Sources/SpeziLLMLocalDownload/LLMLocalDownloadView.swift +++ b/Sources/SpeziLLMLocalDownload/LLMLocalDownloadView.swift @@ -17,7 +17,7 @@ import SwiftUI /// /// It can be combined with the SpeziOnboarding `OnboardingStack` to create an easy onboarding flow within the application. /// -/// The ``LLMLocalDownloadView/init(downloadDescription:llmDownloadUrl:llmStorageUrl:action:)-9hraf`` initializer accepts a download description displayed in the view, the remote download `URL` of the LLM, the local storage `URL` of the downloaded model, as well as an action closure to move onto the next (onboarding) step. +/// The ``LLMLocalDownloadView/init(downloadDescription:llmDownloadUrl:llmStorageUrl:action:)-9hraf`` initializer accepts a download description displayed in the view, the `LLMLocalModel` representing the model to be downloaded, and an action closure to move onto the next (onboarding) step. /// /// The heavy lifting of downloading and storing the model is done by the ``LLMLocalDownloadManager`` which exposes the current downloading state view the ``LLMLocalDownloadManager/state`` property of type ``LLMLocalDownloadManager/DownloadState``. /// @@ -39,9 +39,8 @@ import SwiftUI /// /// var body: some View { /// LLMLocalDownloadView( -/// downloadDescription: "The Llama2 7B model will be downloaded", -/// llmDownloadUrl: LLMLocalDownloadManager.LLMUrlDefaults.llama2ChatModelUrl, // Download the Llama2 7B model -/// llmStorageUrl: .cachesDirectory.appending(path: "llm.gguf") // Store the downloaded LLM in the caches directory +/// model: .llama3_8B_4bit, +/// downloadDescription: "The Llama3 8B model will be downloaded" /// ) { /// onboardingNavigationPath.nextStep() /// } @@ -177,9 +176,8 @@ public struct LLMLocalDownloadView: View { /// Creates a ``LLMLocalDownloadView`` that presents an onboarding view that helps with downloading the necessary LLM files from remote servers. /// /// - Parameters: + /// - model: An `LLMLocalModel` representing the model to download. /// - downloadDescription: Localized description of the to-be-downloaded model shown in the ``LLMLocalDownloadView``. - /// - llmDownloadUrl: The remote `URL` from where the LLM file should be downloaded. - /// - llmDownloadLocation: The local `URL` where the LLM file should be stored. /// - action: The action that should be performed when pressing the primary button of the view. public init( model: LLMLocalModel, @@ -196,9 +194,8 @@ public struct LLMLocalDownloadView: View { /// Creates a ``LLMLocalDownloadView`` that presents an onboarding view that helps with downloading the necessary LLM files from remote servers. /// /// - Parameters: + /// - model: An `LLMLocalModel` representing the model to download. /// - downloadDescription: Description of the to-be-downloaded model shown in the ``LLMLocalDownloadView``. - /// - llmDownloadUrl: The remote `URL` from where the LLM file should be downloaded. - /// - llmDownloadLocation: The local `URL` where the LLM file should be stored. /// - action: The action that should be performed when pressing the primary button of the view. @_disfavoredOverload public init( @@ -218,7 +215,7 @@ public struct LLMLocalDownloadView: View { #if DEBUG #Preview { LLMLocalDownloadView( - model: .phi3_4bit, + model: .llama3_8B_4bit, downloadDescription: "LLM_DOWNLOAD_DESCRIPTION".localized(.module), action: {} ) diff --git a/Sources/SpeziLLMLocalDownload/SpeziLLMLocalDownload.docc/SpeziLLMLocalDownload.md b/Sources/SpeziLLMLocalDownload/SpeziLLMLocalDownload.docc/SpeziLLMLocalDownload.md index ff2358cc..d6827015 100644 --- a/Sources/SpeziLLMLocalDownload/SpeziLLMLocalDownload.docc/SpeziLLMLocalDownload.md +++ b/Sources/SpeziLLMLocalDownload/SpeziLLMLocalDownload.docc/SpeziLLMLocalDownload.md @@ -55,15 +55,16 @@ struct LLMLocalDownloadApp: View { } } } +``` +```swift struct LLMLocalOnboardingDownloadView: View { @Environment(OnboardingNavigationPath.self) private var onboardingNavigationPath var body: some View { LLMLocalDownloadView( - downloadDescription: "The Llama2 7B model will be downloaded", - llmDownloadUrl: LLMLocalDownloadManager.LLMUrlDefaults.llama2ChatModelUrl, // Download the Llama2 7B model - llmStorageUrl: .cachesDirectory.appending(path: "llm.gguf") // Store the downloaded LLM in the caches directory + model: .llama3_8B_4bit, + downloadDescription: "The Llama3 8B model will be downloaded" ) { onboardingNavigationPath.nextStep() } diff --git a/Sources/SpeziLLMOpenAI/FunctionCalling/LLMFunction.swift b/Sources/SpeziLLMOpenAI/FunctionCalling/LLMFunction.swift index 325a929e..fad33df8 100644 --- a/Sources/SpeziLLMOpenAI/FunctionCalling/LLMFunction.swift +++ b/Sources/SpeziLLMOpenAI/FunctionCalling/LLMFunction.swift @@ -51,7 +51,7 @@ /// struct LLMOpenAIChatTestView: View { /// private let schema = LLMOpenAISchema( /// parameters: .init( -/// modelType: .gpt4_turbo, +/// modelType: .gpt4_o, /// systemPrompt: "You're a helpful assistant that answers questions from users." /// ) /// ) { diff --git a/Sources/SpeziLLMOpenAI/SpeziLLMOpenAI.docc/FunctionCalling.md b/Sources/SpeziLLMOpenAI/SpeziLLMOpenAI.docc/FunctionCalling.md index e441f17c..8c12a4f2 100644 --- a/Sources/SpeziLLMOpenAI/SpeziLLMOpenAI.docc/FunctionCalling.md +++ b/Sources/SpeziLLMOpenAI/SpeziLLMOpenAI.docc/FunctionCalling.md @@ -57,7 +57,7 @@ struct WeatherFunction: LLMFunction { struct LLMOpenAIChatTestView: View { private let schema = LLMOpenAISchema( parameters: .init( - modelType: .gpt4_turbo, + modelType: .gpt4_o, systemPrompt: "You're a helpful assistant that answers questions from users." ) ) { diff --git a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+Array.swift b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+Array.swift index 4d4d7384..1bcb8f11 100644 --- a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+Array.swift +++ b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+Array.swift @@ -56,7 +56,7 @@ final class LLMOpenAIParameterArrayTests: XCTestCase { } let llm = LLMOpenAISchema( - parameters: .init(modelType: .gpt4_turbo) + parameters: .init(modelType: .gpt4_o) ) { LLMFunctionTest(someInitArg: "testArg") } diff --git a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+CustomTypes.swift b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+CustomTypes.swift index aac58159..4809bda8 100644 --- a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+CustomTypes.swift +++ b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+CustomTypes.swift @@ -65,7 +65,7 @@ final class LLMOpenAIParameterCustomTypesTests: XCTestCase { } let llm = LLMOpenAISchema( - parameters: .init(modelType: .gpt4_turbo) + parameters: .init(modelType: .gpt4_o) ) { LLMFunctionTest(someInitArg: "testArg") } diff --git a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+Enum.swift b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+Enum.swift index ef294907..6411775d 100644 --- a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+Enum.swift +++ b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+Enum.swift @@ -61,7 +61,7 @@ final class LLMOpenAIParameterEnumTests: XCTestCase { } let llm = LLMOpenAISchema( - parameters: .init(modelType: .gpt4_turbo) + parameters: .init(modelType: .gpt4_o) ) { LLMFunctionTest(someInitArg: "testArg") } diff --git a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+InvalidParameters.swift b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+InvalidParameters.swift index 5761b1c7..34128fc0 100644 --- a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+InvalidParameters.swift +++ b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+InvalidParameters.swift @@ -44,7 +44,7 @@ final class LLMOpenAIInvalidParametersTests: XCTestCase { } let llm = LLMOpenAISchema( - parameters: .init(modelType: .gpt4_turbo) + parameters: .init(modelType: .gpt4_o) ) { LLMFunctionTest(someInitArg: "testArg") } diff --git a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+OptionalTypes.swift b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+OptionalTypes.swift index ac234b6f..ccfd557b 100644 --- a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+OptionalTypes.swift +++ b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+OptionalTypes.swift @@ -75,7 +75,7 @@ final class LLMOpenAIParameterOptionalTypesTests: XCTestCase { } let llm = LLMOpenAISchema( - parameters: .init(modelType: .gpt4_turbo) + parameters: .init(modelType: .gpt4_o) ) { LLMFunctionTest(someInitArg: "testArg") } diff --git a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+PrimitiveTypes.swift b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+PrimitiveTypes.swift index 955ffa31..7def066e 100644 --- a/Tests/SpeziLLMTests/LLMOpenAIParameterTests+PrimitiveTypes.swift +++ b/Tests/SpeziLLMTests/LLMOpenAIParameterTests+PrimitiveTypes.swift @@ -56,7 +56,7 @@ final class LLMOpenAIParameterPrimitiveTypesTests: XCTestCase { } let llm = LLMOpenAISchema( - parameters: .init(modelType: .gpt4_turbo) + parameters: .init(modelType: .gpt4_o) ) { LLMFunctionTest(someInitArg: "testArg") } diff --git a/Tests/UITests/TestApp/LLMOpenAI/LLMOpenAIChatTestView.swift b/Tests/UITests/TestApp/LLMOpenAI/LLMOpenAIChatTestView.swift index 2731f393..f05b67f9 100644 --- a/Tests/UITests/TestApp/LLMOpenAI/LLMOpenAIChatTestView.swift +++ b/Tests/UITests/TestApp/LLMOpenAI/LLMOpenAIChatTestView.swift @@ -15,7 +15,7 @@ import SwiftUI struct LLMOpenAIChatTestView: View { static let schema = LLMOpenAISchema( parameters: .init( - modelType: .gpt4_turbo, + modelType: .gpt4_o, systemPrompt: "You're a helpful assistant that answers questions from users." ) ) { diff --git a/Tests/UITests/TestApp/TestAppDelegate.swift b/Tests/UITests/TestApp/TestAppDelegate.swift index f4cc6369..73e19326 100644 --- a/Tests/UITests/TestApp/TestAppDelegate.swift +++ b/Tests/UITests/TestApp/TestAppDelegate.swift @@ -49,6 +49,7 @@ class TestAppDelegate: SpeziAppDelegate { // No CA certificate (meaning no encrypted traffic) for development purposes, see `caCertificateUrl` above LLMFogPlatform(configuration: .init(host: "spezillmfog.local", caCertificate: nil)) LLMOpenAIPlatform() + LLMLocalPlatform() // Note: Spezi LLM Local is not compatible with simulators. } } }