Merge pull request #24 from Codium-ai/docs-context

Docs context

.github/workflows/ci.yml

@@ -1,8 +1,8 @@
-name: ci 
+name: ci
 on:
   push:
     branches:
-      - master 
+      - master
       - main
 permissions:
   contents: write
@@ -25,7 +25,7 @@ jobs:
           path: .cache
           restore-keys: |
             mkdocs-material-
-      - run: pip install mkdocs-material 
+      - run: pip install mkdocs-material
       - run: pip install "mkdocs-material[imaging]"
       - run: pip install mkdocs-glightbox
       - run: pip install mkdocs-video

.github/workflows/docs-to-proxy.yml

@@ -0,0 +1,99 @@
+name: docs-to-proxy
+on: workflow_dispatch
+
+
+jobs:
+  update-docs-context:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Run build script
+        run: python scripts/build_docs_context.py # this will create a .llm_context.txt file in repo-root/.llm_context.txt
+
+      - name: Clone codium-proxy
+        uses: actions/checkout@v2
+        with:
+          repository: Codium-ai/codium-proxy
+          ref: main
+          path: codium-proxy
+          ssh-key: ${{secrets.CODIUM_PROXY_SSH}}
+
+      - name: Make updates to codium-proxy docs content
+        working-directory: codium-proxy
+        run: |
+          # write .llm_context content to documentation/docs_context.txt
+          echo "$(cat ../.llm_context.txt)" > documentation/docs_context.txt
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b update-docs-context-branch
+          git add documentation/docs_context.txt
+          git commit -m "Automated update to docs_context.txt"
+
+      - name: Push branch to remote
+        id: push-branch
+        continue-on-error: true  # In case the branch fails we want to notify users on slack.
+        working-directory: codium-proxy
+        run: |
+          git push -u origin update-docs-context-branch
+
+      - name: Send Slack Failure Notification
+        if: steps.push-branch.outcome == 'failure'
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        run: |
+          curl -X POST -H 'Content-type: application/json' \
+          --data "{\"text\":\"A update-docs branch already exists. Please delete it and re-run the workflow.\"}" \
+          $SLACK_WEBHOOK_URL
+
+      - name: Stop workflow on failure
+        if: steps.push-branch.outcome == 'failure'
+        run: |
+          echo "Stopping workflow due to an existing update-docs branch."
+          exit 1
+
+      - name: Wait for branch to propagate
+        run: sleep 5
+
+      - name: Create pull request
+        id: create_pr
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.CODIUM_PROXY_TOKEN }} # This is YoavO personal access token need to move to a github app.
+          script: |
+            try {
+              console.log('Creating pull request...');
+              const pullRequest = await github.rest.pulls.create({
+                owner: 'Codium-ai',
+                repo: 'codium-proxy',
+                title: 'Automated update to docs_context.txt',
+                head: 'update-docs-context-branch',
+                base: 'main',
+                body: 'This is an automated pull request to update docs_context.txt',
+              });
+              console.log('Pull request created:', pullRequest.data.html_url);
+              core.setOutput("pull_request_url", pullRequest.data.html_url);
+            } catch (error) {
+              console.log('Error details:', {
+                message: error.message,
+                status: error.status,
+                response: error.response ? error.response.data : null
+              });
+              core.setFailed(error.message);
+            }
+
+      - name: Send Slack Notification
+        if: success()
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          PULL_REQUEST_URL: ${{ steps.create_pr.outputs.pull_request_url }}
+        run: |
+          curl -X POST -H 'Content-type: application/json' \
+          --data "{\"text\":\"Documentation context is updated please approve PR: ${PULL_REQUEST_URL}\"}" \
+          $SLACK_WEBHOOK_URL

scripts/build_docs_context.py

@@ -0,0 +1,53 @@
+#! /usr/bin/env python3
+
+from pathlib import Path
+import os
+import re
+
+CONTEXT_FILE_NAME = ".llm_context.txt"
+DOCS_DIRECTORY = "docs"
+
+
+def get_markdown_content(directory):
+    """
+    Recursively find and concatenate all markdown files in a directory.
+    Uses some ad-hoc logic to remove different elemnts:
+     - Images
+     - Examples.
+     - UI tags.
+    
+    Args:
+        directory (str): Path to the directory to search
+        
+    Returns:
+        str: Concatenated content of all markdown files
+    """
+    markdown_content = ""
+
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith(('.md', '.markdown')):
+                file_path = os.path.join(root, file)
+                try:
+                    with open(file_path, 'r', encoding='utf-8') as f:
+                        # Remove all html tags
+                        # Stop reading once you encounter a line that starts with a !!!
+                        for line in f:
+                            if line.startswith("!!!"):
+                                break
+                            line_content = re.sub(r'<[^>]*>', '', line)
+                            line_content = re.sub(r'\{.*?\}', '', line_content)
+                            markdown_content += line_content
+
+                except Exception as e:
+                    print(f"Error reading {file_path}: {str(e)}")
+
+    # Remove all more than 1 newlines
+    markdown_content = re.sub(r'\n{3,}', '\n', markdown_content)
+    return markdown_content
+
+if __name__ == "__main__":
+    docs_path = Path(__file__).parent.parent / DOCS_DIRECTORY
+    markdown_content = get_markdown_content(docs_path)
+    with open(CONTEXT_FILE_NAME, "w") as f:
+        f.write(markdown_content)