Merge branch 'xiaoyi_doc' into li

.github/workflows/documentation.yml

@@ -0,0 +1,83 @@
+name: Documentation
+
+on:
+  push:
+    branches:
+      - xiaoyi_doc  # Ensure this is the branch where you commit documentation updates
+
+permissions:
+  contents: write
+  actions: read
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Ensure full git history is fetched
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Install dependencies using Poetry
+        run: |
+          poetry config virtualenvs.create false
+          poetry install
+
+      - name: List installed packages
+        run: |
+          poetry run pip list
+
+      - name: Print Sphinx version
+        run: |
+          poetry run sphinx-build --version
+
+      - name: Build documentation
+        run: |
+          echo "Current Working Directory: $(pwd)"
+          echo "Python path before Sphinx build: $PYTHONPATH"
+          poetry run sphinx-build -b html ./docs/source/ ./docs/build/ -vvv
+          echo "Listing detailed contents of build directory:"
+          find ./docs/build/ -type f
+
+      - name: Test module import
+        run: |
+          poetry run python -c "import lightrag; print('Lightrag module loaded from:', lightrag.__file__)"
+
+      - name: Print effective Sphinx conf
+        run: |
+          poetry run python -c "from sphinx.config import Config; config = Config.read('./docs/source/conf.py'); print(config.values)"
+
+      - name: Check API documentation files
+        run: |
+          echo "Checking API documentation directory for components:"
+          ls -la ./docs/build/apis/components/
+
+      - name: Create .nojekyll file
+        run: |
+          touch ./docs/build/.nojekyll
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_branch: gh-pages
+          publish_dir: ./docs/build/
+          user_name: github-actions[bot]
+          user_email: github-actions[bot]@users.noreply.github.com
+
+      - name: Debug Output
+        run: |
+          pwd  # Print the current working directory
+          ls -l ./docs/build/  # List files in the build directory
+          cat ./docs/source/conf.py  # Show Sphinx config file for debugging

docs/requirements.txt

@@ -1,4 +1,11 @@
-pydata-sphinx-theme==0.15.2
-Sphinx==7.3.7
-sphinx_design==0.6.0
-sphinx-copybutton==0.5.2
+pydata-sphinx-theme==0.15.3
+sphinx-design==0.6.0
+sphinx-copybutton==0.5.2
+sphinx==7.3.7
+nbsphinx==0.9.4
+nbconvert==7.16.4
+PyYAML
+readthedocs-sphinx-search==0.3.2
+numpy
+tqdm
+tiktoken

docs/source/apis/components/index.rst

@@ -9,9 +9,10 @@ Overview
 
    components.agent
    components.model_client
+   components.data_process
 
    .. components.reasoning
-
+   
    components.retriever
    components.output_parsers
 
@@ -37,6 +38,13 @@ Model Clients
 
    components.model_client
 
+Data Process
+----------------
+.. toctree::
+   :maxdepth: 1
+
+   components.data_process
+
 .. Embedders
 .. ---------
 .. .. toctree::

docs/source/apis/core/index.rst

@@ -7,22 +7,19 @@ Overview
 ----------
 .. autosummary::
 
-   core.base_data_class
-   core.model_client
+   core.base_data_class  
    core.component
-   core.data_components
    core.db
    core.default_prompt_template
-   core.document_splitter
    core.embedder
    core.functional
    core.generator
    core.memory
+   core.model_client
    core.parameter
    core.prompt_builder
    core.retriever
    core.string_parser
-   core.text_splitter
    core.tokenizer
    core.func_tool
    core.tool_manager
@@ -51,8 +48,6 @@ Data Handling
    core.base_data_class
    core.types
 
-
-   core.data_components
    core.db
 
 Prompts and Templates
@@ -63,10 +58,10 @@ Prompts and Templates
    core.default_prompt_template
    core.prompt_builder
 
-Document Processing
--------------------
-.. toctree::
-   :maxdepth: 1
+.. Document Processing
+.. -------------------
+.. .. toctree::
+..    :maxdepth: 1
 
    .. core.document_splitter
    core.text_splitter

docs/source/apis/index.rst

@@ -17,7 +17,6 @@ The core section of the LightRAG API documentation provides detailed information
    core.data_components
    core.db
    core.default_prompt_template
-   core.document_splitter
    core.embedder
    core.functional
    core.generator
@@ -26,7 +25,6 @@ The core section of the LightRAG API documentation provides detailed information
    core.prompt_builder
    core.retriever
    core.string_parser
-   core.text_splitter
    core.tokenizer
    core.func_tool
    core.tool_manager
@@ -42,9 +40,9 @@ The components section of the LightRAG API documentation outlines the detailed s
 
    components.agent
    components.model_client
-
+   componnets.data_process
    .. components.reasoning
-
+   
    components.retriever
    components.output_parsers
 

docs/source/conf.py

@@ -28,7 +28,6 @@
 copyright = "2024, SylphAI, Inc"
 author = "SylphAI, Inc"
 
-
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be