From 4ee00a7565841541ecb25d1dc43931da7fef41e1 Mon Sep 17 00:00:00 2001 From: Ben Zimmerman <59181438+T3CH3Y@users.noreply.github.com> Date: Thu, 16 Nov 2023 15:42:48 -0500 Subject: [PATCH 01/14] reorg --- mint.json | 51 ++++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/mint.json b/mint.json index f9bfef7..f1f7792 100644 --- a/mint.json +++ b/mint.json @@ -34,7 +34,7 @@ "name": "Home" }, "topAnchor": { - "name": "Documentation", + "name": "Anarchy LLM-VM", "icon": "graduation-cap" }, "anchors": [ @@ -66,7 +66,30 @@ ], "navigation": [ { - "group": "Welcome", + "group": "Get Started", + "pages":[ + { + "group": "๐Ÿ„ Quickstart", + "pages": [ + "get_started/system", + "get_started/install", + "get_started/quickstart/completions" + ] + }, + { + "group":"๐Ÿƒ Advanced LLM-VM", + "pages":[ + "get_started/quickstart/finetuning", + "get_started/quickstart/agents", + "get_started/quickstart/local_llms", + "get_started/quickstart/server" + ] + }, + "welcome/roadmap" + ] + }, + { + "group": "About Us", "pages": [ "welcome/why_anarchy", "welcome/llm_vm", @@ -75,29 +98,7 @@ "welcome/contact" ] }, - { - "group": "LLM-VM", - "pages":[ - { - "group": "๐Ÿ„ Quickstart", - "pages": [ - "get_started/system", - "get_started/install", - "get_started/quickstart/completions" - ] - }, - { - "group":"๐Ÿƒ Advanced LLM-VM", - "pages":[ - "get_started/quickstart/finetuning", - "get_started/quickstart/agents", - "get_started/quickstart/local_llms", - "get_started/quickstart/server" - ] - }, - "welcome/roadmap" - ] - }, + { "group": "Getting Involved", "pages": [ From 5ae19f4f0c23b3683b46a50c5ea09ed6311d2eb8 Mon Sep 17 00:00:00 2001 From: Ben Zimmerman <59181438+T3CH3Y@users.noreply.github.com> Date: Thu, 16 Nov 2023 15:45:37 -0500 Subject: [PATCH 02/14] removed blog --- mint.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mint.json b/mint.json index f1f7792..c846473 100644 --- a/mint.json +++ b/mint.json @@ -56,12 +56,6 @@ "icon": "shirt", "color": "#1100bb", "url": "https://swag.anarchy.ai" - }, - { - "name": "Blog", - "icon": "bullhorn", - "color": "#1100bb", - "url": "https://anarchy.ai/blog" } ], "navigation": [ From 19b67530dc312a6580b6dcd7b4d07fbcb7a51708 Mon Sep 17 00:00:00 2001 From: "business@benzimmerman.org" Date: Thu, 16 Nov 2023 17:15:09 -0500 Subject: [PATCH 03/14] cutting the fat --- blog.mdx | 11 ----------- mint.json | 39 ++++++++++++++++++++++++--------------- overview/introduction.mdx | 4 ++++ 3 files changed, 28 insertions(+), 26 deletions(-) delete mode 100644 blog.mdx create mode 100644 overview/introduction.mdx diff --git a/blog.mdx b/blog.mdx deleted file mode 100644 index 800deb1..0000000 --- a/blog.mdx +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: 'Blog' -description: "Posts about the great work we are doing at Anarchy" ---- - -## Posts - -- **[Guided Distillation](https://anarchy.ai/blog/guided_distillation)** Read about how we finetuned a Pythia model to constrain the output space of a 70 million parameter Pythia model! - -- **[The REBEL Agent](https://anarchy.ai/blog/rebel)** Read about the REBEL agent, a tool that empowers LLMs to use external tools to solve compositional questions! - diff --git a/mint.json b/mint.json index c846473..04fcf27 100644 --- a/mint.json +++ b/mint.json @@ -60,8 +60,8 @@ ], "navigation": [ { - "group": "Get Started", - "pages":[ + "group": "Overview", + "pages":["overview/introduction", { "group": "๐Ÿ„ Quickstart", "pages": [ @@ -70,6 +70,26 @@ "get_started/quickstart/completions" ] }, + "welcome/roadmap", + { + "group":"About Us", + "pages":[ + + "welcome/why_anarchy", + "welcome/llm_vm", + "welcome/mission", + "welcome/who_are_we", + "welcome/contact" + + ] + } + + ] + }, + { + "group": "Features", + "pages":[ + { "group":"๐Ÿƒ Advanced LLM-VM", "pages":[ @@ -78,25 +98,14 @@ "get_started/quickstart/local_llms", "get_started/quickstart/server" ] - }, - "welcome/roadmap" + } ] }, - { - "group": "About Us", - "pages": [ - "welcome/why_anarchy", - "welcome/llm_vm", - "welcome/mission", - "welcome/who_are_we", - "welcome/contact" - ] - }, + { "group": "Getting Involved", "pages": [ - "contributing/getting_involved", "contributing/jobs", "contributing/community_development", "contributing/constitution" diff --git a/overview/introduction.mdx b/overview/introduction.mdx new file mode 100644 index 0000000..cbc8a94 --- /dev/null +++ b/overview/introduction.mdx @@ -0,0 +1,4 @@ +--- +title: "Introduction" +description: "What is Anarchy LLM-VM?" +--- \ No newline at end of file From 40b53a9fc33245a6c1f2321b63eaaecec5491f4d Mon Sep 17 00:00:00 2001 From: "business@benzimmerman.org" Date: Thu, 16 Nov 2023 17:34:53 -0500 Subject: [PATCH 04/14] some minute distro changes --- mint.json | 2 +- {welcome => overview}/roadmap.mdx | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {welcome => overview}/roadmap.mdx (100%) diff --git a/mint.json b/mint.json index 04fcf27..53b4428 100644 --- a/mint.json +++ b/mint.json @@ -70,7 +70,7 @@ "get_started/quickstart/completions" ] }, - "welcome/roadmap", + "overview/roadmap", { "group":"About Us", "pages":[ diff --git a/welcome/roadmap.mdx b/overview/roadmap.mdx similarity index 100% rename from welcome/roadmap.mdx rename to overview/roadmap.mdx From 9984ab1d50a9afb22490df3d67e7e1a0a8670a7f Mon Sep 17 00:00:00 2001 From: "business@benzimmerman.org" Date: Tue, 28 Nov 2023 19:20:29 -0500 Subject: [PATCH 05/14] straight forward tabs --- mint.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mint.json b/mint.json index 53b4428..3b31048 100644 --- a/mint.json +++ b/mint.json @@ -58,6 +58,18 @@ "url": "https://swag.anarchy.ai" } ], + + "tabs": [ + {"name": "LLM-VM", + "url":"llmvm"}, + {"name": "Chat.Dev", + "url":"chatdev"}, + {"name": "VecQL", + "url":"vecql"} + + + ], + "navigation": [ { "group": "Overview", From 5a27c226afcc6cdeca2ef4601527f2d602f61a80 Mon Sep 17 00:00:00 2001 From: "business@benzimmerman.org" Date: Wed, 29 Nov 2023 02:04:00 -0500 Subject: [PATCH 06/14] overhaul --- .../quickstart => llmvm/features}/agents.mdx | 0 .../features}/finetuning.mdx | 0 .../features}/local_llms.mdx | 0 .../quickstart => llmvm/features}/server.mdx | 0 mint.json | 26 +++++++++------ products/chat_dev.mdx | 32 +++++++++++++++++++ products/llm_vm.mdx | 32 +++++++++++++++++++ products/vecql.mdx | 32 +++++++++++++++++++ 8 files changed, 112 insertions(+), 10 deletions(-) rename {get_started/quickstart => llmvm/features}/agents.mdx (100%) rename {get_started/quickstart => llmvm/features}/finetuning.mdx (100%) rename {get_started/quickstart => llmvm/features}/local_llms.mdx (100%) rename {get_started/quickstart => llmvm/features}/server.mdx (100%) create mode 100644 products/chat_dev.mdx create mode 100644 products/llm_vm.mdx create mode 100644 products/vecql.mdx diff --git a/get_started/quickstart/agents.mdx b/llmvm/features/agents.mdx similarity index 100% rename from get_started/quickstart/agents.mdx rename to llmvm/features/agents.mdx diff --git a/get_started/quickstart/finetuning.mdx b/llmvm/features/finetuning.mdx similarity index 100% rename from get_started/quickstart/finetuning.mdx rename to llmvm/features/finetuning.mdx diff --git a/get_started/quickstart/local_llms.mdx b/llmvm/features/local_llms.mdx similarity index 100% rename from get_started/quickstart/local_llms.mdx rename to llmvm/features/local_llms.mdx diff --git a/get_started/quickstart/server.mdx b/llmvm/features/server.mdx similarity index 100% rename from get_started/quickstart/server.mdx rename to llmvm/features/server.mdx diff --git a/mint.json b/mint.json index 3b31048..60e0b7e 100644 --- a/mint.json +++ b/mint.json @@ -99,20 +99,26 @@ ] }, { - "group": "Features", + "group": "Products", "pages":[ - - { + "products/chat_dev", + "products/llm_vm", + "products/vecql" + ] + }, + {"group": "Features", + "pages": [{ "group":"๐Ÿƒ Advanced LLM-VM", "pages":[ - "get_started/quickstart/finetuning", - "get_started/quickstart/agents", - "get_started/quickstart/local_llms", - "get_started/quickstart/server" + "llmvm/features/finetuning", + "llmvm/features/agents", + "llmvm/features/local_llms", + "llmvm/features/server" ] - } - ] - }, + }, + + +]}, { diff --git a/products/chat_dev.mdx b/products/chat_dev.mdx new file mode 100644 index 0000000..87afc79 --- /dev/null +++ b/products/chat_dev.mdx @@ -0,0 +1,32 @@ +--- +title: 'Chat.Dev' +description: 'The Large Language Model Virtual Machine(LLM-VM) is a optimized backend for managing LLMs. It serves to facilitate communication and coordination between data, language models(running on the CPU), prompts, and various tools to streamline AGI development.' +--- + +## Features of the LLM-VM +- **Implicit Agents** - The Anarchy LLM-VM can be set up to use external tools through our agents such as [REBEL](../get_started/quickstart/agents) just by supplying tool descriptions! + +- **Inference Optimization** - The Anarchy LLM-VM is optimized from agent level all the way to assembly on known LLM architectures to get the most bang for your buck. With state of the art batching, sparse inference and quantization, distillation, and multi-level colocation, we aim to provide the fastest framework available. + +- **Task Auto-Optimization** - The Anarchy LLM-VM will analyze your use cases for repetitive tasks where it can activate student-teacher distillation to train a super-efficient small model from a larger more general model without loosing accuracy. It can furthermore take advantage of data-synthesis techniques to improve results. + +- **Library Callable** - Our library can be used from any python codebase directly. + +- **HTTP Endpoint** - We've provided an HTTP standalone server to handle completion requests via a convenient API. + + +## Why use our LLM-VM? + +- **Speed up development** - With Anarchy, one interface is all you need to interact with the latest LLMs available. + +- **Lower costs** - Running models locally can reduce the pay-as-you-go costs of development and testing. + +- **Flexibility** - Anarchy allows you to rapidly switch between popular models so you can pinpoint the exact right tool for your project. + +- **Community** - Join our active community of highly motivated developers and engineers working passionately to democratize AGI + +## Goals +The LLM-VM wants to centralize and optimize the functionalities of modern completion endpoints in an opinionated way, allowing for the efficient batching of calls that might otherwise be extremely costly across multiple endpoints + +We want to make the LLM-VM model and architecture agnostic. We want to create a backend that gives you an optimized solution regardless of which model you choose, and which architecture and hardware solution you choose to run it on + diff --git a/products/llm_vm.mdx b/products/llm_vm.mdx new file mode 100644 index 0000000..e9f15f7 --- /dev/null +++ b/products/llm_vm.mdx @@ -0,0 +1,32 @@ +--- +title: 'LLM-VM' +description: 'The Large Language Model Virtual Machine(LLM-VM) is a optimized backend for managing LLMs. It serves to facilitate communication and coordination between data, language models(running on the CPU), prompts, and various tools to streamline AGI development.' +--- + +## Features of the LLM-VM +- **Implicit Agents** - The Anarchy LLM-VM can be set up to use external tools through our agents such as [REBEL](../get_started/quickstart/agents) just by supplying tool descriptions! + +- **Inference Optimization** - The Anarchy LLM-VM is optimized from agent level all the way to assembly on known LLM architectures to get the most bang for your buck. With state of the art batching, sparse inference and quantization, distillation, and multi-level colocation, we aim to provide the fastest framework available. + +- **Task Auto-Optimization** - The Anarchy LLM-VM will analyze your use cases for repetitive tasks where it can activate student-teacher distillation to train a super-efficient small model from a larger more general model without loosing accuracy. It can furthermore take advantage of data-synthesis techniques to improve results. + +- **Library Callable** - Our library can be used from any python codebase directly. + +- **HTTP Endpoint** - We've provided an HTTP standalone server to handle completion requests via a convenient API. + + +## Why use our LLM-VM? + +- **Speed up development** - With Anarchy, one interface is all you need to interact with the latest LLMs available. + +- **Lower costs** - Running models locally can reduce the pay-as-you-go costs of development and testing. + +- **Flexibility** - Anarchy allows you to rapidly switch between popular models so you can pinpoint the exact right tool for your project. + +- **Community** - Join our active community of highly motivated developers and engineers working passionately to democratize AGI + +## Goals +The LLM-VM wants to centralize and optimize the functionalities of modern completion endpoints in an opinionated way, allowing for the efficient batching of calls that might otherwise be extremely costly across multiple endpoints + +We want to make the LLM-VM model and architecture agnostic. We want to create a backend that gives you an optimized solution regardless of which model you choose, and which architecture and hardware solution you choose to run it on + diff --git a/products/vecql.mdx b/products/vecql.mdx new file mode 100644 index 0000000..508b6b2 --- /dev/null +++ b/products/vecql.mdx @@ -0,0 +1,32 @@ +--- +title: 'VecQL' +description: 'The Large Language Model Virtual Machine(LLM-VM) is a optimized backend for managing LLMs. It serves to facilitate communication and coordination between data, language models(running on the CPU), prompts, and various tools to streamline AGI development.' +--- + +## Features of the LLM-VM +- **Implicit Agents** - The Anarchy LLM-VM can be set up to use external tools through our agents such as [REBEL](../get_started/quickstart/agents) just by supplying tool descriptions! + +- **Inference Optimization** - The Anarchy LLM-VM is optimized from agent level all the way to assembly on known LLM architectures to get the most bang for your buck. With state of the art batching, sparse inference and quantization, distillation, and multi-level colocation, we aim to provide the fastest framework available. + +- **Task Auto-Optimization** - The Anarchy LLM-VM will analyze your use cases for repetitive tasks where it can activate student-teacher distillation to train a super-efficient small model from a larger more general model without loosing accuracy. It can furthermore take advantage of data-synthesis techniques to improve results. + +- **Library Callable** - Our library can be used from any python codebase directly. + +- **HTTP Endpoint** - We've provided an HTTP standalone server to handle completion requests via a convenient API. + + +## Why use our LLM-VM? + +- **Speed up development** - With Anarchy, one interface is all you need to interact with the latest LLMs available. + +- **Lower costs** - Running models locally can reduce the pay-as-you-go costs of development and testing. + +- **Flexibility** - Anarchy allows you to rapidly switch between popular models so you can pinpoint the exact right tool for your project. + +- **Community** - Join our active community of highly motivated developers and engineers working passionately to democratize AGI + +## Goals +The LLM-VM wants to centralize and optimize the functionalities of modern completion endpoints in an opinionated way, allowing for the efficient batching of calls that might otherwise be extremely costly across multiple endpoints + +We want to make the LLM-VM model and architecture agnostic. We want to create a backend that gives you an optimized solution regardless of which model you choose, and which architecture and hardware solution you choose to run it on + From 1948b86070e55104b99807cd23c1b0010e16c4c4 Mon Sep 17 00:00:00 2001 From: "business@benzimmerman.org" Date: Wed, 29 Nov 2023 02:15:15 -0500 Subject: [PATCH 07/14] Basic Structural Changes --- llmvm/features/awq.mdx | 49 ++++++++++++++++++++++++++++++++++++++ llmvm/features/rag.mdx | 49 ++++++++++++++++++++++++++++++++++++++ llmvm/features/rebel.mdx | 49 ++++++++++++++++++++++++++++++++++++++ llmvm/features/selfins.mdx | 49 ++++++++++++++++++++++++++++++++++++++ llmvm/features/synth.mdx | 49 ++++++++++++++++++++++++++++++++++++++ mint.json | 30 ++++++++++++++--------- 6 files changed, 264 insertions(+), 11 deletions(-) create mode 100644 llmvm/features/awq.mdx create mode 100644 llmvm/features/rag.mdx create mode 100644 llmvm/features/rebel.mdx create mode 100644 llmvm/features/selfins.mdx create mode 100644 llmvm/features/synth.mdx diff --git a/llmvm/features/awq.mdx b/llmvm/features/awq.mdx new file mode 100644 index 0000000..7d3fec0 --- /dev/null +++ b/llmvm/features/awq.mdx @@ -0,0 +1,49 @@ +--- +title: AWQ +description: 'Our AI agents expand what you can do with LLMs!' +--- + +## The REBEL Agent +**REcursion Based Extensible LLM** + +Our REBEL agent takes a novel approach to answering complex questions. Using recursive reasoning, REBEL expands what LLMs can do with problem decomposition and tool use. In this way, we are able to answer questions requiring data LLMs were not directly trained on. + +### Running REBEL +**Getting started with REBEL is easy** +```python quickstart_REBEL.py +# import our client +from llm_vm.client import Client +import os + +# Instantiate the client specifying which LLM you want to use +client = Client(big_model='chat_gpt', small_model='gpt') #REBEL will use chat_gpt no matter what big model is specified here, this specification exists for non-REBEL completion calls. + +#Calling client.complete with a tool list specified leads to the REBEL agent being used. +response = client.complete( + prompt = 'Is it warmer in Paris or Timbuktu and what are the temperatures in either city?', + context='', + openai_key=os.getenv("OPENAI_API_KEY"), #for REBEL we need an OpenAI key + tools=[{'description': 'Find the weather at a location and returns it in celcius.', #this tool list contains only one dictionary, therefore only one tool + 'dynamic_params': { + "latitude": 'latitude of as a float', + "longitude": 'the longitude as a float' + }, + 'method': 'GET', + 'url': "https://api.open-meteo.com/v1/forecast", + 'static_params': {'current_weather': 'true'}}]) #No tools by default, so you have to add your own +print(response) +``` +### Tool Definition +Tools are defined by dictionaries that are added to the list ```tools```. These dictionaries need to contain the following fields: + +|Field| Type | Description| +|-|-|-| +|```'description'```| string | A description of what the tool does| +|```'dynamic_params'```| dictionary | A dictionary containing key value pairs (paramter name : description) of the API endpoint's mutable parameters that need to be set by REBEL in order to answer a query| +|```'method'```| string | ```GET``` or ```POST```, whichever is the type of the API endpoint| +|```'url'```| string | URL of the API endpoint that the given tool specifies| +|```'static_params'```| dictionary | Any parameters that are constant between all API calls. An API key/token is an example of this| + +You can add any tool you want and as many as you need. REBEL will attempt to compositionally answer a question that may require calling multiple tools. + + diff --git a/llmvm/features/rag.mdx b/llmvm/features/rag.mdx new file mode 100644 index 0000000..cc30d64 --- /dev/null +++ b/llmvm/features/rag.mdx @@ -0,0 +1,49 @@ +--- +title: RAG +description: 'Our AI agents expand what you can do with LLMs!' +--- + +## The REBEL Agent +**REcursion Based Extensible LLM** + +Our REBEL agent takes a novel approach to answering complex questions. Using recursive reasoning, REBEL expands what LLMs can do with problem decomposition and tool use. In this way, we are able to answer questions requiring data LLMs were not directly trained on. + +### Running REBEL +**Getting started with REBEL is easy** +```python quickstart_REBEL.py +# import our client +from llm_vm.client import Client +import os + +# Instantiate the client specifying which LLM you want to use +client = Client(big_model='chat_gpt', small_model='gpt') #REBEL will use chat_gpt no matter what big model is specified here, this specification exists for non-REBEL completion calls. + +#Calling client.complete with a tool list specified leads to the REBEL agent being used. +response = client.complete( + prompt = 'Is it warmer in Paris or Timbuktu and what are the temperatures in either city?', + context='', + openai_key=os.getenv("OPENAI_API_KEY"), #for REBEL we need an OpenAI key + tools=[{'description': 'Find the weather at a location and returns it in celcius.', #this tool list contains only one dictionary, therefore only one tool + 'dynamic_params': { + "latitude": 'latitude of as a float', + "longitude": 'the longitude as a float' + }, + 'method': 'GET', + 'url': "https://api.open-meteo.com/v1/forecast", + 'static_params': {'current_weather': 'true'}}]) #No tools by default, so you have to add your own +print(response) +``` +### Tool Definition +Tools are defined by dictionaries that are added to the list ```tools```. These dictionaries need to contain the following fields: + +|Field| Type | Description| +|-|-|-| +|```'description'```| string | A description of what the tool does| +|```'dynamic_params'```| dictionary | A dictionary containing key value pairs (paramter name : description) of the API endpoint's mutable parameters that need to be set by REBEL in order to answer a query| +|```'method'```| string | ```GET``` or ```POST```, whichever is the type of the API endpoint| +|```'url'```| string | URL of the API endpoint that the given tool specifies| +|```'static_params'```| dictionary | Any parameters that are constant between all API calls. An API key/token is an example of this| + +You can add any tool you want and as many as you need. REBEL will attempt to compositionally answer a question that may require calling multiple tools. + + diff --git a/llmvm/features/rebel.mdx b/llmvm/features/rebel.mdx new file mode 100644 index 0000000..cf60a8b --- /dev/null +++ b/llmvm/features/rebel.mdx @@ -0,0 +1,49 @@ +--- +title: REBEL +description: 'Our AI agents expand what you can do with LLMs!' +--- + +## The REBEL Agent +**REcursion Based Extensible LLM** + +Our REBEL agent takes a novel approach to answering complex questions. Using recursive reasoning, REBEL expands what LLMs can do with problem decomposition and tool use. In this way, we are able to answer questions requiring data LLMs were not directly trained on. + +### Running REBEL +**Getting started with REBEL is easy** +```python quickstart_REBEL.py +# import our client +from llm_vm.client import Client +import os + +# Instantiate the client specifying which LLM you want to use +client = Client(big_model='chat_gpt', small_model='gpt') #REBEL will use chat_gpt no matter what big model is specified here, this specification exists for non-REBEL completion calls. + +#Calling client.complete with a tool list specified leads to the REBEL agent being used. +response = client.complete( + prompt = 'Is it warmer in Paris or Timbuktu and what are the temperatures in either city?', + context='', + openai_key=os.getenv("OPENAI_API_KEY"), #for REBEL we need an OpenAI key + tools=[{'description': 'Find the weather at a location and returns it in celcius.', #this tool list contains only one dictionary, therefore only one tool + 'dynamic_params': { + "latitude": 'latitude of as a float', + "longitude": 'the longitude as a float' + }, + 'method': 'GET', + 'url': "https://api.open-meteo.com/v1/forecast", + 'static_params': {'current_weather': 'true'}}]) #No tools by default, so you have to add your own +print(response) +``` +### Tool Definition +Tools are defined by dictionaries that are added to the list ```tools```. These dictionaries need to contain the following fields: + +|Field| Type | Description| +|-|-|-| +|```'description'```| string | A description of what the tool does| +|```'dynamic_params'```| dictionary | A dictionary containing key value pairs (paramter name : description) of the API endpoint's mutable parameters that need to be set by REBEL in order to answer a query| +|```'method'```| string | ```GET``` or ```POST```, whichever is the type of the API endpoint| +|```'url'```| string | URL of the API endpoint that the given tool specifies| +|```'static_params'```| dictionary | Any parameters that are constant between all API calls. An API key/token is an example of this| + +You can add any tool you want and as many as you need. REBEL will attempt to compositionally answer a question that may require calling multiple tools. + + diff --git a/llmvm/features/selfins.mdx b/llmvm/features/selfins.mdx new file mode 100644 index 0000000..e3f0230 --- /dev/null +++ b/llmvm/features/selfins.mdx @@ -0,0 +1,49 @@ +--- +title: Self Instruct +description: 'Our AI agents expand what you can do with LLMs!' +--- + +## The REBEL Agent +**REcursion Based Extensible LLM** + +Our REBEL agent takes a novel approach to answering complex questions. Using recursive reasoning, REBEL expands what LLMs can do with problem decomposition and tool use. In this way, we are able to answer questions requiring data LLMs were not directly trained on. + +### Running REBEL +**Getting started with REBEL is easy** +```python quickstart_REBEL.py +# import our client +from llm_vm.client import Client +import os + +# Instantiate the client specifying which LLM you want to use +client = Client(big_model='chat_gpt', small_model='gpt') #REBEL will use chat_gpt no matter what big model is specified here, this specification exists for non-REBEL completion calls. + +#Calling client.complete with a tool list specified leads to the REBEL agent being used. +response = client.complete( + prompt = 'Is it warmer in Paris or Timbuktu and what are the temperatures in either city?', + context='', + openai_key=os.getenv("OPENAI_API_KEY"), #for REBEL we need an OpenAI key + tools=[{'description': 'Find the weather at a location and returns it in celcius.', #this tool list contains only one dictionary, therefore only one tool + 'dynamic_params': { + "latitude": 'latitude of as a float', + "longitude": 'the longitude as a float' + }, + 'method': 'GET', + 'url': "https://api.open-meteo.com/v1/forecast", + 'static_params': {'current_weather': 'true'}}]) #No tools by default, so you have to add your own +print(response) +``` +### Tool Definition +Tools are defined by dictionaries that are added to the list ```tools```. These dictionaries need to contain the following fields: + +|Field| Type | Description| +|-|-|-| +|```'description'```| string | A description of what the tool does| +|```'dynamic_params'```| dictionary | A dictionary containing key value pairs (paramter name : description) of the API endpoint's mutable parameters that need to be set by REBEL in order to answer a query| +|```'method'```| string | ```GET``` or ```POST```, whichever is the type of the API endpoint| +|```'url'```| string | URL of the API endpoint that the given tool specifies| +|```'static_params'```| dictionary | Any parameters that are constant between all API calls. An API key/token is an example of this| + +You can add any tool you want and as many as you need. REBEL will attempt to compositionally answer a question that may require calling multiple tools. + + diff --git a/llmvm/features/synth.mdx b/llmvm/features/synth.mdx new file mode 100644 index 0000000..4b2783a --- /dev/null +++ b/llmvm/features/synth.mdx @@ -0,0 +1,49 @@ +--- +title: Synthetic Distillation +description: 'Our AI agents expand what you can do with LLMs!' +--- + +## The REBEL Agent +**REcursion Based Extensible LLM** + +Our REBEL agent takes a novel approach to answering complex questions. Using recursive reasoning, REBEL expands what LLMs can do with problem decomposition and tool use. In this way, we are able to answer questions requiring data LLMs were not directly trained on. + +### Running REBEL +**Getting started with REBEL is easy** +```python quickstart_REBEL.py +# import our client +from llm_vm.client import Client +import os + +# Instantiate the client specifying which LLM you want to use +client = Client(big_model='chat_gpt', small_model='gpt') #REBEL will use chat_gpt no matter what big model is specified here, this specification exists for non-REBEL completion calls. + +#Calling client.complete with a tool list specified leads to the REBEL agent being used. +response = client.complete( + prompt = 'Is it warmer in Paris or Timbuktu and what are the temperatures in either city?', + context='', + openai_key=os.getenv("OPENAI_API_KEY"), #for REBEL we need an OpenAI key + tools=[{'description': 'Find the weather at a location and returns it in celcius.', #this tool list contains only one dictionary, therefore only one tool + 'dynamic_params': { + "latitude": 'latitude of as a float', + "longitude": 'the longitude as a float' + }, + 'method': 'GET', + 'url': "https://api.open-meteo.com/v1/forecast", + 'static_params': {'current_weather': 'true'}}]) #No tools by default, so you have to add your own +print(response) +``` +### Tool Definition +Tools are defined by dictionaries that are added to the list ```tools```. These dictionaries need to contain the following fields: + +|Field| Type | Description| +|-|-|-| +|```'description'```| string | A description of what the tool does| +|```'dynamic_params'```| dictionary | A dictionary containing key value pairs (paramter name : description) of the API endpoint's mutable parameters that need to be set by REBEL in order to answer a query| +|```'method'```| string | ```GET``` or ```POST```, whichever is the type of the API endpoint| +|```'url'```| string | URL of the API endpoint that the given tool specifies| +|```'static_params'```| dictionary | Any parameters that are constant between all API calls. An API key/token is an example of this| + +You can add any tool you want and as many as you need. REBEL will attempt to compositionally answer a question that may require calling multiple tools. + + diff --git a/mint.json b/mint.json index 60e0b7e..904ede3 100644 --- a/mint.json +++ b/mint.json @@ -107,18 +107,26 @@ ] }, {"group": "Features", - "pages": [{ - "group":"๐Ÿƒ Advanced LLM-VM", - "pages":[ - "llmvm/features/finetuning", - "llmvm/features/agents", - "llmvm/features/local_llms", - "llmvm/features/server" - ] - }, - + "pages": [ + "llmvm/features/synth", + "llmvm/features/rag", + "llmvm/features/rebel", + "llmvm/features/selfins", + "llmvm/features/awq" + + + ]}, + + { + "group":"๐Ÿƒ Advanced", + "pages":[ + "llmvm/features/finetuning", + "llmvm/features/agents", + "llmvm/features/local_llms", + "llmvm/features/server" + ] + }, -]}, { From 7ddd15ba0ebd515d3fa09225b806bd6db06ff90f Mon Sep 17 00:00:00 2001 From: Kenny <35975075+al1y3vk@users.noreply.github.com> Date: Wed, 29 Nov 2023 15:20:46 -0600 Subject: [PATCH 08/14] Editing the RAG feature documentation --- llmvm/features/rag.mdx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llmvm/features/rag.mdx b/llmvm/features/rag.mdx index cc30d64..5735317 100644 --- a/llmvm/features/rag.mdx +++ b/llmvm/features/rag.mdx @@ -1,14 +1,14 @@ --- title: RAG -description: 'Our AI agents expand what you can do with LLMs!' +description: 'RAG models combine the powers of pretrained dense retrieval (DPR) and sequence-to-sequence models.' --- -## The REBEL Agent -**REcursion Based Extensible LLM** +## The RAG Model +**Retrieval-augmented generation** -Our REBEL agent takes a novel approach to answering complex questions. Using recursive reasoning, REBEL expands what LLMs can do with problem decomposition and tool use. In this way, we are able to answer questions requiring data LLMs were not directly trained on. +RAG models retrieve documents, pass them to a seq2seq model, then marginalize to generate outputs. The retriever and seq2seq modules are initialized from pretrained models, and fine-tuned jointly, allowing both retrieval and generation to adapt to downstream tasks. -### Running REBEL +### Running RAG **Getting started with REBEL is easy** ```python quickstart_REBEL.py # import our client From 1c245cacd754b0c9a1bbbf797527149c3b3ecf13 Mon Sep 17 00:00:00 2001 From: Kenny <35975075+al1y3vk@users.noreply.github.com> Date: Wed, 29 Nov 2023 16:16:20 -0600 Subject: [PATCH 09/14] Adding AWQ documentation --- llmvm/features/awq.mdx | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/llmvm/features/awq.mdx b/llmvm/features/awq.mdx index 7d3fec0..d3380a0 100644 --- a/llmvm/features/awq.mdx +++ b/llmvm/features/awq.mdx @@ -1,12 +1,18 @@ --- title: AWQ -description: 'Our AI agents expand what you can do with LLMs!' +description: 'AWQ takes the concept of weight quantization to the next level by considering the activations of the model during the quantization process.' --- -## The REBEL Agent -**REcursion Based Extensible LLM** +## The Role of AWQ +**Activation-Aware Weight Quantization** -Our REBEL agent takes a novel approach to answering complex questions. Using recursive reasoning, REBEL expands what LLMs can do with problem decomposition and tool use. In this way, we are able to answer questions requiring data LLMs were not directly trained on. +In traditional weight quantization, the weights are quantized independently of the data they process. In AWQ, the quantization process takes into account the actual data distribution in the activations produced by the model during inference. + +Here's how: + +1. Collect Activation Statistics: During a calibration phase, a subset of the data is used to collect statistics on the activations produced by the model. This involves running the model on this data and recording the range of values and the distribution of activations. +2. Searching Weight Quantization Parameters: Weights are quantized by taking the activation statistics into account. Concretely, we perform a space search for quantization paremeters (e.g., scales and zeropoints), to minimize the distortions incurred by quantization on output activations. As a result, the quantized weights can be accurately represented with fewer bits. +3. Quantizing: With the quantization parameters in place, the model weights are quantized using a reduced number of bits. ### Running REBEL **Getting started with REBEL is easy** From a562164e3a4748413218673560e3b7ea3f35ca31 Mon Sep 17 00:00:00 2001 From: Kenny <35975075+al1y3vk@users.noreply.github.com> Date: Wed, 29 Nov 2023 16:38:20 -0600 Subject: [PATCH 10/14] Added documentation for self-instruct --- llmvm/features/selfins.mdx | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/llmvm/features/selfins.mdx b/llmvm/features/selfins.mdx index e3f0230..2a2dd29 100644 --- a/llmvm/features/selfins.mdx +++ b/llmvm/features/selfins.mdx @@ -1,13 +1,18 @@ --- -title: Self Instruct -description: 'Our AI agents expand what you can do with LLMs!' +title: SELF-INSTRUCT +description: 'Motivation for SELF-INSTRUCT is to reduce the dependence on human annotators.' --- -## The REBEL Agent -**REcursion Based Extensible LLM** +## The SELF-INSTRUCTor +**The non-human annotator** -Our REBEL agent takes a novel approach to answering complex questions. Using recursive reasoning, REBEL expands what LLMs can do with problem decomposition and tool use. In this way, we are able to answer questions requiring data LLMs were not directly trained on. +Large language models can be effective at following instructions if tuned with annotated โ€œinstructionalโ€ data. However, this needs a large dataset of human-generated instructional commands and their desired instances. Creating a large dataset via humans is expensive. Furthermore, creating such a human-generated dataset can lack diversity and creativity. +SELF-INSTRUCT tackles this bottleneck by reducing the dependence on human annotators. At a high level, there are two components: + +1. Starting with a seed set of instructions+instances, the self-instruct process uses LLMs to generate new instructions and instances in a bootstrapped manner. + +2. Next, it uses the generated instructions to finetune an instruction-following model from a vanilla pre-trained language model. ### Running REBEL **Getting started with REBEL is easy** ```python quickstart_REBEL.py From b15a93e1b2dbd922fefb65cca03b10224c9b6a33 Mon Sep 17 00:00:00 2001 From: Kenny <35975075+al1y3vk@users.noreply.github.com> Date: Wed, 29 Nov 2023 17:27:21 -0600 Subject: [PATCH 11/14] Added documentation for Synthetic Dystillation --- llmvm/features/synth.mdx | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/llmvm/features/synth.mdx b/llmvm/features/synth.mdx index 4b2783a..6449ff4 100644 --- a/llmvm/features/synth.mdx +++ b/llmvm/features/synth.mdx @@ -1,12 +1,32 @@ --- title: Synthetic Distillation -description: 'Our AI agents expand what you can do with LLMs!' +description: 'The early version of Synthetic Distillation used a generator model trained adversarially against the student model to generate synthetic data to train the student model.' --- -## The REBEL Agent -**REcursion Based Extensible LLM** +## The Synthetic Distiller +**The artificial purifier** -Our REBEL agent takes a novel approach to answering complex questions. Using recursive reasoning, REBEL expands what LLMs can do with problem decomposition and tool use. In this way, we are able to answer questions requiring data LLMs were not directly trained on. +There are three new ways to generate data synthetically: + +1. Random sampling +Synthetic data are generated by sampling randomly from an input distribution. To obtain the best +results, the synthetic data should be drawn from a similar distribution as the actual data. +In the case where actual input data have been standardized, random samples can be drawn from a +Gaussian distribution ~ N (0, I). However in other cases, where the input has been scaled differently, +or has clearly defined input space boundaries, it is important to confirm the data distribution and +input space boundaries, and design the sampling method such that samples conform to the actual +input distribution as much as possible. The input distribution and input space bounds may be defined +using the available validation or test set or based upon some prior knowledge. + +2. Generator model +In this method, a generator model parameterized by a specific variable is trained to output samples +that would result in a large difference between the student and teacher model's predictions. This +generator model is trained in an adversarial manner against the student model during the distillation +process by optimizing the generator loss function. The student is trained using the student loss to minimize the difference between teacher and its own +predictions, the two opposing learning objectives are trained in a sequential adversarial manner, and +the student model is able to learn to match the predictions of the teacher model as training continues. + +3. Direct optimization from random samples. To learn about the third method (which is more complicated), and to read everything in detail, check out the original source: https://arxiv.org/abs/2301.04338 ### Running REBEL **Getting started with REBEL is easy** From 1fad996c643967ec599988bb3c1b14f1df430386 Mon Sep 17 00:00:00 2001 From: bilal-aamer Date: Wed, 13 Dec 2023 23:01:19 +0530 Subject: [PATCH 12/14] fix broken discord invite links --- mint.json | 2 +- welcome/contact.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mint.json b/mint.json index 904ede3..9ab838d 100644 --- a/mint.json +++ b/mint.json @@ -139,7 +139,7 @@ } ], "footerSocials": { - "discord": "https://discord.com/invite/qaFf7S373c", + "discord": "https://discord.anarchy.ai/", "github": "https://github.com/anarchy-ai/llm-vm", "hacker-news": "https://www.ycombinator.com/companies/anarchy", "twitter": "https://twitter.com/anarchy_ai_inc", diff --git a/welcome/contact.mdx b/welcome/contact.mdx index 4c3b68f..1985b79 100644 --- a/welcome/contact.mdx +++ b/welcome/contact.mdx @@ -5,7 +5,7 @@ description: "There are a number of ways to contact us depending on your needs" ## Quickest way -Hop on our [discord](https://discord.gg/qaFf7S373c)! +Hop on our [discord](https://discord.anarchy.ai/)! ## Customer From 5a8d4ff673e89fb53604c1dfb2b670f4bf1a86ee Mon Sep 17 00:00:00 2001 From: bilal-aamer Date: Wed, 13 Dec 2023 23:40:17 +0530 Subject: [PATCH 13/14] Updated roadmap --- overview/roadmap.mdx | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/overview/roadmap.mdx b/overview/roadmap.mdx index 9754213..cf673aa 100644 --- a/overview/roadmap.mdx +++ b/overview/roadmap.mdx @@ -7,12 +7,32 @@ description: "This project is in BETA and developing rapidly. We're eager to get **We're actively working on delivering these features** -- **Live Data Augmentation** - You will be able to provide a live updating data-set and the Anarchy LLM-VM will fine-tune your models or work with a vector DB to provide up-to-date information with citations +- **Live Data Augmentation** -- You will be able to provide a live updating data-set and the Anarchy LLM-VM will fine-tune your models or work with a vector DB to provide up-to-date information with citations -- **Web Playground** - You will be able to run the Anarchy LLM-VM and test it's outputs from the browser. +- **Web Playground** -- You will be able to run the Anarchy LLM-VM and test it's outputs from the browser. - **Load-Balancing and Orchestration** -- If you have multiple LLMs or providers you'd like to utilize, you will be able to hand them to the Anarchy LLM-VM to automatically figure out which to work with and when to optimize your uptime or your costs -- **Output Templating** - You can ensure that the LLM only outputs data in specific formats and fills in variables from a template with either regular expressions, LMQL, or OpenAI's template language +- **Output Templating** -- You can ensure that the LLM only outputs data in specific formats and fills in variables from a template with either regular expressions, LMQL, or OpenAI's template language - **Persistent Stateful Memory** - The Anarchy LLM-VM can remember a user's conversation history and react accordingly + +- **Implicit Agents ๐Ÿ”ง๐Ÿ•ต๏ธ** -- The Anarchy LLM-VM can be set up to use external tools through our agents such as REBEL just by supplying tool descriptions! + +- **Inference Optimization ๐Ÿš„** -- The Anarchy LLM-VM is optimized from the agent level all the way to assembly on known LLM architectures to get the most bang for your buck. With state-of-the-art batching, sparse inference and quantization, distillation, and multi-level colocation, we aim to provide the fastest framework available. + +- **Task Auto-Optimization ๐Ÿš…** -- The Anarchy LLM-VM will analyze your use cases for repetitive tasks where it can activate student-teacher distillation to train a super-efficient small model from a larger more general model without losing accuracy. It can furthermore take advantage of data-synthesis techniques to improve results. + +- **Library Callable ๐Ÿ“š** -- We provide a library that can be used from any Python codebase directly. + +- **HTTP Endpoints ๐Ÿ•ธ๏ธ** -- We provide an HTTP standalone server to handle completion requests. + +- **Smart batching ๐Ÿ—ž๏ธ** -- Handle multiple calls at the same time from different levels of the llm-vm + +- **Speculative Preemptive Sampling ๐Ÿ”ฎ** -- Use a small LLM to predict outputs of a larger LLM and don't fall back to the large one unless sampling is getting bad. + +- **Token Streaming ๐Ÿšฐ** -- Get a hook for a constantly updating supply of tokens! + +- **Streamed Backtracking ๐Ÿ”™** -- Didn't like one output? Look at others! Efficiently. + +NOTE: Do refer our pinned ticket on [Architecture Overview & Roadmap](https://github.com/anarchy-ai/LLM-VM/issues/277) \ No newline at end of file From bb6dac2e355a20ebe0da97d7a6c3d30095490d37 Mon Sep 17 00:00:00 2001 From: bilal-aamer Date: Thu, 21 Dec 2023 12:33:05 +0530 Subject: [PATCH 14/14] add client docs --- llmvm/features/client.mdx | 128 ++++++++++++++++++++++++++++++++++++++ mint.json | 1 + 2 files changed, 129 insertions(+) create mode 100644 llmvm/features/client.mdx diff --git a/llmvm/features/client.mdx b/llmvm/features/client.mdx new file mode 100644 index 0000000..33760da --- /dev/null +++ b/llmvm/features/client.mdx @@ -0,0 +1,128 @@ +The Anarchy Client is a Python library that facilitates easy access to Anarchy, a large language model (LLM) optimized for generating text-based completions. It provides a convenient interface to interact with the LLM, perform fine-tuning, and utilize various tools and agents for text generation. The client includes two main classes: `Simple_Inference_Client` and `Client`. + +## Simple Inference Client + +The `Simple_Inference_Client` is designed for straightforward text generation without the need for fine-tuning or additional agents. It allows you to generate text completions using the specified Anarchy LLM model. + +### Usage + +```python +from anarchy_client import Simple_Inference_Client + +# Initialize the Simple Inference Client +client = Simple_Inference_Client(model="chat_gpt", openai_key="YOUR_OPENAI_API_KEY") + +# Generate text completion +response = client.complete(prompt="Generate text based on this prompt.") +print(response) +``` + +### Methods + +`complete(prompt, max_len=256, **kwargs)` + +Generate text completion based on the provided prompt. + +- `prompt` (str): The input prompt for text generation. +- `max_len` (int, optional): The maximum length of the generated text. Default is 256. +- `**kwargs` (dict, optional): Additional keyword arguments for text generation. + +## Client + +The `Client` class offers more advanced features, including fine-tuning, data synthesis, and integration with agents for specialized text generation tasks. + +### Usage + +```python +from anarchy_client import Client + +# Initialize the Client with default models +client = Client() + +# Generate text completion +response = client.complete(prompt="Generate text based on this prompt.") +print(response) +``` + +### Initialization Parameters + +- `big_model` (str, optional): Name of the reliable source Anarchy LLM model. Default is `"chat_gpt"`. +- `small_model` (str, optional): Name of the small model used for fine-tuning. Default is `"pythia"`. +- `big_model_config` (dict, optional): Configuration options for the reliable source model. +- `small_model_config` (dict, optional): Configuration options for the small model used in fine-tuning. + +### Methods + +`complete(prompt, context="", openai_key=None, finetune=False, data_synthesis=False, temperature=0, stoptoken=None, tools=None, openai_kwargs={}, hf_kwargs={})` + +Generate text completion using the Anarchy LLM. + +- `prompt` (str): The input prompt for text generation. +- `context` (str, optional): Unchanging context to send to the LLM for generation. Defaults to an empty string and does not perform fine-tuning. +- `openai_key` (str, optional): API key for OpenAI access. +- `finetune` (bool, optional): If `True`, fine-tuning is initiated. Default is `False`. +- `data_synthesis` (bool, optional): Boolean value to determine whether data should be synthesized for fine-tuning. Default is `False`. +- `temperature` (float, optional): Sampling temperature for text generation, ranging from 0 to 2. +- `stoptoken` (str or list, optional): Sequence for stopping token generation. +- `tools` (list, optional): List of API tools for use with the Rebel agent. +- `openai_kwargs` (dict, optional): Keyword arguments for generation with OpenAI models. +- `hf_kwargs` (dict, optional): Keyword arguments for generation with Hugging Face models. + +`load_finetune(model_filename=None, small_model=False)` + +Load a fine-tuned model for either the reliable source (big) model or the small model. + +- `model_filename` (str, optional): The filename of the fine-tuned model. +- `small_model` (bool, optional): If `True`, load the model for the small model. Default is `False`. + +`change_model_dtype(big_model_dtype=None, small_model_dtype=None)` + +Change the model data type for either the reliable source (big) model or the small model. + +- `big_model_dtype` (str, optional): Data type for the reliable source model. +- `small_model_dtype` (str, optional): Data type for the small model. + +`set_pinecone_db(api_key, env_name)` + +Set up the connection to a Pinecone vector database for storing and querying embeddings. + +- `api_key` (str): API key for accessing the Pinecone database. +- `env_name` (str): Name of the Pinecone environment. + +`store_pdf(pdf_file_path, chunk_size=1024, **kwargs)` + +Store text data from a PDF file into the Pinecone vector database after encoding it into embeddings. + +- `pdf_file_path` (str): Path to the PDF file. +- `chunk_size` (int, optional): Chunk size for processing the PDF. Default is 1024. +- `**kwargs` (dict, optional): Additional keyword arguments for storing data in the database. + +`create_pinecone_index(**kwargs)` + +Create an index in the Pinecone vector database. + +- `**kwargs` (dict, optional): Additional keyword arguments for creating the index. + +`RAG_complete(prompt, context="", openai_key=None, finetune=False, data_synthesis=False, temperature=0, stoptoken=None, tools=None, query_kwargs={}, hf_kwargs={}, openai_kwargs={})` + +Generate text completion using the RAG (Retrieval-Augmented Generation) approach. This method first retrieves relevant documents from the Pinecone database based on the prompt's embeddings and then generates text completions. + +- `prompt` (str): The input prompt for text generation. +- `context` (str, optional): Unchanging context to send to the LLM for generation. Defaults to an empty string and does not perform fine-tuning. +- `openai_key` (str, optional): API key for OpenAI access. +- `finetune` (bool, optional): If `True`, fine-tuning is initiated. Default is `False`. +- `data_synthesis` (bool, optional): Boolean value to determine whether data should be synthesized for fine-tuning. Default is `False`. +- `temperature` (float, optional): Sampling temperature for text generation, ranging from 0 to 2. +- `stoptoken` (str or list, optional): Sequence for stopping token generation. +- `tools` (list, optional): List of API tools for use with the Rebel agent. +- `query_kwargs` (dict, optional): Keyword arguments for querying the Pinecone vector database. +- `hf_kwargs` (dict, optional): Keyword arguments for generation with Hugging Face models. +- `openai_kwargs` (dict, optional): Keyword arguments for generation with OpenAI models. + +## Examples + +For detailed examples and usage of the Anarchy Client, refer to the provided code snippets and the Anarchy Client documentation. + +## License + +This library is provided under the MIT License. See the [LICENSE](LICENSE) file for more details. \ No newline at end of file diff --git a/mint.json b/mint.json index 9ab838d..e617d4a 100644 --- a/mint.json +++ b/mint.json @@ -108,6 +108,7 @@ }, {"group": "Features", "pages": [ + "llmvm/features/client", "llmvm/features/synth", "llmvm/features/rag", "llmvm/features/rebel",