diff --git a/docs/Makefile b/docs/Makefile index d0c3cbf1..1478681e 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,12 +1,14 @@ # Minimal makefile for Sphinx documentation # -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = LightRAG +SOURCEDIR = source# the source of output and conf.py BUILDDIR = build +APIDOCOUTDIR = $(SOURCEDIR)/apis # Put it first so that "make" without argument is like "make help". help: @@ -18,3 +20,14 @@ help: # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +apidoc: + @echo "Running sphinx-apidoc for core" + @sphinx-apidoc -o $(APIDOCOUTDIR)/core ../core --separate + @echo "Running sphinx-apidoc for components" + @sphinx-apidoc -o $(APIDOCOUTDIR)/components ../components --separate + + + +html: apidoc + @$(SPHINXBUILD) -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index 745abcde..e03d3149 100644 --- a/docs/README.md +++ b/docs/README.md @@ -170,4 +170,42 @@ make html Remember to exclude any unnecessary files in `.gitignore`. Please don’t commit files in `docs/build`. We can dynamically build local documentation with the make files and `source/`. -Please push your updates to the GitHub repo. \ No newline at end of file +Please push your updates to the GitHub repo. + +The structure of the code base and the docs: + +``` +LightRAG/ +├── docs/ +│ ├── apis/ +│ │ ├── core/ +│ │ │ ├── core.module1.rst +│ │ │ ├── core.module2.rst +│ │ ├── components/ +│ │ │ ├── components.module1.rst +│ │ │ ├── components.module2.rst +│ ├── build/ +│ │ ├── html/ +│ │ │ ├── _static/ +│ │ │ ├── _templates/ +│ │ │ ├── index.html +│ │ │ ├── core/ +│ │ │ │ ├── core.module1.html +│ │ │ │ ├── core.module2.html +│ │ │ ├── components/ +│ │ │ │ ├── components.module1.html +│ │ │ │ ├── components.module2.html +│ ├── _static/ +│ ├── _templates/ +│ ├── conf.py +│ ├── index.rst +│ ├── Makefile +├── core/ +│ ├── __init__.py +│ ├── module1.py +│ ├── module2.py +├── components/ +│ ├── __init__.py +│ ├── module1.py +│ ├── module2.py +``` \ No newline at end of file diff --git a/docs/source/apis/components/components.agent.rst b/docs/source/apis/components/components.agent.rst deleted file mode 100644 index 82a27f88..00000000 --- a/docs/source/apis/components/components.agent.rst +++ /dev/null @@ -1,10 +0,0 @@ -LightRAG.components.agent -========================= - -components.agent.react\_agent ------------------------------------- - -.. automodule:: components.agent.react_agent - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/apis/components/components.api_client.rst b/docs/source/apis/components/components.api_client.rst deleted file mode 100644 index 6c2e9484..00000000 --- a/docs/source/apis/components/components.api_client.rst +++ /dev/null @@ -1,20 +0,0 @@ -LightRAG.components.api\_client -=============================== - - -components.api\_client.groq\_client ------------------------------------------- - -.. automodule:: components.api_client.groq_client - :members: - :undoc-members: - :show-inheritance: - -components.api\_client.transformers\_client --------------------------------------------------- - -.. automodule:: components.api_client.transformers_client - :members: - :undoc-members: - :show-inheritance: - diff --git a/docs/source/apis/components/components.embedder.rst b/docs/source/apis/components/components.embedder.rst deleted file mode 100644 index f2459a35..00000000 --- a/docs/source/apis/components/components.embedder.rst +++ /dev/null @@ -1,3 +0,0 @@ -LightRAG.components.embedder -============================ - diff --git a/docs/source/apis/components/components.reasoning.rst b/docs/source/apis/components/components.reasoning.rst deleted file mode 100644 index 0eac74a1..00000000 --- a/docs/source/apis/components/components.reasoning.rst +++ /dev/null @@ -1,13 +0,0 @@ -LightRAG.components.reasoning -============================= - - - -components.reasoning.chain\_of\_thought ----------------------------------------------- - -.. automodule:: components.reasoning.chain_of_thought - :members: - :undoc-members: - :show-inheritance: - diff --git a/docs/source/apis/components/components.retriever.rst b/docs/source/apis/components/components.retriever.rst deleted file mode 100644 index 8081d58a..00000000 --- a/docs/source/apis/components/components.retriever.rst +++ /dev/null @@ -1,3 +0,0 @@ -LightRAG.components.retriever -============================= - diff --git a/docs/source/apis/components/components.rst b/docs/source/apis/components/components.rst deleted file mode 100644 index 1d848d48..00000000 --- a/docs/source/apis/components/components.rst +++ /dev/null @@ -1,14 +0,0 @@ -LightRAG.components -=================== - - -.. toctree:: - :maxdepth: 4 - - components.agent - components.api_client - components.embedder - components.reasoning - components.retriever - - diff --git a/docs/source/apis/core/core.rst b/docs/source/apis/core/core.rst deleted file mode 100644 index a23e0df1..00000000 --- a/docs/source/apis/core/core.rst +++ /dev/null @@ -1,140 +0,0 @@ -LightRAG.core -============= - - -core.api\_client ------------------------ - -.. automodule:: core.api_client - :members: - :undoc-members: - :show-inheritance: - -core.component ---------------------- - -.. automodule:: core.component - :members: - :undoc-members: - :show-inheritance: - -core.data\_classes -------------------------- - -.. automodule:: core.data_classes - :members: - :undoc-members: - :show-inheritance: - -core.data\_components ----------------------------- - -.. automodule:: core.data_components - :members: - :undoc-members: - :show-inheritance: - -core.db --------------- - -.. automodule:: core.db - :members: - :undoc-members: - :show-inheritance: - -core.document\_splitter ------------------------------- - -.. automodule:: core.document_splitter - :members: - :undoc-members: - :show-inheritance: - -core.documents\_data\_class ----------------------------------- - -.. automodule:: core.documents_data_class - :members: - :undoc-members: - :show-inheritance: - -core.embedder --------------------- - -.. automodule:: core.embedder - :members: - :undoc-members: - :show-inheritance: - -core.functional ----------------------- - -.. automodule:: core.functional - :members: - :undoc-members: - :show-inheritance: - -core.generator ---------------------- - -.. automodule:: core.generator - :members: - :undoc-members: - :show-inheritance: - -core.memory ------------------- - -.. automodule:: core.memory - :members: - :undoc-members: - :show-inheritance: - -core.openai\_client --------------------------- - -.. automodule:: core.openai_client - :members: - :undoc-members: - :show-inheritance: - -core.prompt\_builder ---------------------------- - -.. automodule:: core.prompt_builder - :members: - :undoc-members: - :show-inheritance: - -core.retriever ---------------------- - -.. automodule:: core.retriever - :members: - :undoc-members: - :show-inheritance: - -core.string\_parser --------------------------- - -.. automodule:: core.string_parser - :members: - :undoc-members: - :show-inheritance: - -core.tokenizer ---------------------- - -.. automodule:: core.tokenizer - :members: - :undoc-members: - :show-inheritance: - -core.tool\_helper ------------------------- - -.. automodule:: core.tool_helper - :members: - :undoc-members: - :show-inheritance: - diff --git a/docs/source/conf.py b/docs/source/conf.py index 1cf40cc8..2ad5b33e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,10 +13,15 @@ import os import sys -sys.path.insert(0, os.path.abspath(".")) -sys.path.insert(0, os.path.abspath("..")) -sys.path.insert(0, os.path.abspath("../..")) +# sys.path.insert(0, os.path.abspath(".")) +# sys.path.insert(0, os.path.abspath("..")) +# sys.path.insert(0, os.path.abspath("../..")) sys.path.insert(0, os.path.abspath("../../")) +sys.path.insert(0, os.path.abspath("../../core")) +sys.path.insert(0, os.path.abspath("../../components")) +# sys.path.insert(0, os.path.abspath("..")) +# sys.path.insert(0, os.path.abspath("../core")) +# sys.path.insert(0, os.path.abspath("../components")) # -- Project information ----------------------------------------------------- @@ -41,6 +46,7 @@ "sphinx.ext.viewcode", "sphinx.ext.githubpages", "sphinx.ext.todo", + "sphinx.ext.autosectionlabel", # 'recommonmark', # 'myst_parser' ] @@ -51,6 +57,7 @@ # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [] + # exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**.module.rst', '**/tests/*', '**/test_*.py', '*test.rst'] # -- Options for HTML output ------------------------------------------------- @@ -83,4 +90,5 @@ "member-order": "bysource", "show-inheritance": True, "undoc-members": True, + "autosectionlabel_prefix_document": True, } diff --git a/docs/source/developer_notes/prompt.rst b/docs/source/developer_notes/prompt.rst index 4e734713..754be46f 100644 --- a/docs/source/developer_notes/prompt.rst +++ b/docs/source/developer_notes/prompt.rst @@ -1,12 +1,12 @@ -class Prompt [link to api class] +Prompt [link to api class] ============ LightRAG library in default maximizes developers' control towards the final experience and performance, simplify the development process, and minimize the token consumption. For the major chat models, we eventually will only send two messages to the model: the system message and the user message. The user message is simple, -often you have a message `{'role': 'user', 'text': 'Hello, how are you?'}`. The system message is more complex, it contains the task description, tools, examples, chat history, context, and -intermediate step history from the agent. +often you have a message `{'role': 'user', 'content': 'Hello, how are you?'}`. The system message is more complex, it contains the task description, tools, examples, chat history, context, and +intermediate step history from agents. -Our `DEFAULT_LIGHTRAG_PROMPT` decides the content you send to the system and is represented with `jinjia2` template with 6 variables: `task_desc_str`, `tools_str`, `examples_str`, `chat_history_str`, `context_str`, and `steps_str`. +Our `DEFAULT_LIGHTRAG_SYSTEM_PROMPT` templates the system prompt with 6 important sections. We leverage `jinjia2` template with 6 variables: `task_desc_str`, `tools_str`, `examples_str`, `chat_history_str`, `context_str`, and `steps_str`. .. code-block:: python :linenos: @@ -59,8 +59,14 @@ Across our library, here our advanced features: - Agent with multiple step planning and replanning capabilities, where the `steps_str` variable is used to pass the previous steps to the model. -Note: this means in default our out-of-box components would not support API providers's tools/function calls as we only send the system and user messages to the model. -But it should not stop you from implementing them yourself. +**Note: this means in default our out-of-box components would not support API providers's tools/function calls as we only send the system and user messages to the model. +But it should not stop you from implementing them yourself.** + +Prompt class +--------------------- +We designed a `Prompt` class to render the `template` with the variables to string as the final system prompt. In the simplest case, the string is empty and we will only send +a user message to the model. And in most cases, you want to add at least the `task_desc_str` to the system message. + Prompt and Special Tokens context --------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 6c7bb98c..9f198db4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -45,12 +45,14 @@ LightRAG comes from the best of the AI research and engineering. Fundamentally, tutorials/simpleQA + .. toctree:: :maxdepth: 1 :caption: API Reference - apis/components/components - apis/core/core + apis/core/modules + apis/components/modules + .. toctree:: :glob: