allow saving of file into dir

AnswerDotAI · Sep 13, 2024 · ef3718b · ef3718b
1 parent 5a26fba
commit ef3718b
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 56 deletions.
diff --git a/llms_txt/_modidx.py b/llms_txt/_modidx.py
@@ -10,7 +10,6 @@
                                'llms_txt.core._parse_llms': ('core.html#_parse_llms', 'llms_txt/core.py'),
                                'llms_txt.core._section': ('core.html#_section', 'llms_txt/core.py'),
                                'llms_txt.core.create_ctx': ('core.html#create_ctx', 'llms_txt/core.py'),
-                               'llms_txt.core.find_root_dir': ('core.html#find_root_dir', 'llms_txt/core.py'),
                                'llms_txt.core.get_doc_content': ('core.html#get_doc_content', 'llms_txt/core.py'),
                                'llms_txt.core.get_sizes': ('core.html#get_sizes', 'llms_txt/core.py'),
                                'llms_txt.core.llms_txt2ctx': ('core.html#llms_txt2ctx', 'llms_txt/core.py'),

diff --git a/llms_txt/core.py b/llms_txt/core.py
@@ -3,8 +3,8 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_core.ipynb.
 
 # %% auto 0
-__all__ = ['opt_re', 'named_re', 'search', 'parse_link', 'parse_llms_file', 'find_root_dir', 'get_doc_content', 'mk_ctx',
-           'get_sizes', 'create_ctx', 'llms_txt2ctx']
+__all__ = ['opt_re', 'named_re', 'search', 'parse_link', 'parse_llms_file', 'get_doc_content', 'mk_ctx', 'get_sizes',
+           'create_ctx', 'llms_txt2ctx']
 
 # %% ../nbs/01_core.ipynb
 import re
@@ -15,6 +15,7 @@
 from fastcore.script import *
 import httpx
 from urllib.parse import urlparse
+from nbdev.config import get_config
 
 # %% ../nbs/01_core.ipynb
 def opt_re(s):
@@ -67,29 +68,14 @@ def parse_llms_file(txt):
 # %% ../nbs/01_core.ipynb
 from fastcore.xml import Sections,Project,Doc
 
-# %% ../nbs/01_core.ipynb
-def find_root_dir():
-    "Find the root directory of the nbdev project by looking for settings.ini"
-    path = Path.cwd()
-    while path != path.parent:
-        if (path / 'settings.ini').exists(): return path
-        path = path.parent
-    return None
-
 # %% ../nbs/01_core.ipynb
 def get_doc_content(url):
     "Fetch content from local file if in nbdev repo."
-    root_dir = find_root_dir()
-    if root_dir:
-        config = Config(root_dir, 'settings.ini')
-        doc_host = config.get('doc_host')
-        if doc_host and url.startswith(doc_host):
-            parsed_url = urlparse(url)
-            relative_path = parsed_url.path.lstrip('/')
-            local_path = root_dir / '_docs' / relative_path
-            if local_path.exists():
-                with open(local_path, 'r') as f: return f.read()
-    # If not a local file or file doesn't exist, fetch from URL
+    cfg = get_config()
+    if url.startswith(cfg.doc_host):
+        relative_path = urlparse(url).path.lstrip('/')
+        local_path = cfg.doc_path / relative_path
+        if local_path.exists(): return local_path.read_text()
     return httpx.get(url).text
 
 # %% ../nbs/01_core.ipynb
@@ -131,7 +117,10 @@ def create_ctx(txt, optional=False, n_workers=None):
 def llms_txt2ctx(
     fname:str, # File name to read
     optional:bool_arg=False, # Include 'optional' section?
-    n_workers:int=None # Number of threads to use for parallel downloading
+    n_workers:int=None, # Number of threads to use for parallel downloading
+    save_nbdev:bool_arg=False #save output to nbdev `docs_path` instead of emitting to stdout
 ):
     "Print a `Project` with a `Section` for each H2 part in file read from `fname`, optionally skipping the 'optional' section."
-    print(create_ctx(Path(fname).read_text(), optional=optional, n_workers=n_workers))
+    ctx = create_ctx(Path(fname).read_text(), optional=optional, n_workers=n_workers)
+    if save_nbdev: (get_config().doc_path / fname).write_text(ctx)
+    else: print(ctx)
diff --git a/nbs/01_core.ipynb b/nbs/01_core.ipynb
@@ -50,7 +50,8 @@
     "from fastcore.xml import *\n",
     "from fastcore.script import *\n",
     "import httpx\n",
-    "from urllib.parse import urlparse"
+    "from urllib.parse import urlparse\n",
+    "from nbdev.config import get_config"
    ]
   },
   {
@@ -674,22 +675,6 @@
     "from fastcore.xml import Sections,Project,Doc"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "#| export\n",
-    "def find_root_dir():\n",
-    "    \"Find the root directory of the nbdev project by looking for settings.ini\"\n",
-    "    path = Path.cwd()\n",
-    "    while path != path.parent:\n",
-    "        if (path / 'settings.ini').exists(): return path\n",
-    "        path = path.parent\n",
-    "    return None"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -699,17 +684,11 @@
     "#|export\n",
     "def get_doc_content(url):\n",
     "    \"Fetch content from local file if in nbdev repo.\"\n",
-    "    root_dir = find_root_dir()\n",
-    "    if root_dir:\n",
-    "        config = Config(root_dir, 'settings.ini')\n",
-    "        doc_host = config.get('doc_host')\n",
-    "        if doc_host and url.startswith(doc_host):\n",
-    "            parsed_url = urlparse(url)\n",
-    "            relative_path = parsed_url.path.lstrip('/')\n",
-    "            local_path = root_dir / '_docs' / relative_path\n",
-    "            if local_path.exists():\n",
-    "                with open(local_path, 'r') as f: return f.read()\n",
-    "    # If not a local file or file doesn't exist, fetch from URL\n",
+    "    cfg = get_config()\n",
+    "    if url.startswith(cfg.doc_host):\n",
+    "        relative_path = urlparse(url).path.lstrip('/')\n",
+    "        local_path = cfg.doc_path / relative_path\n",
+    "        if local_path.exists(): return local_path.read_text()\n",
     "    return httpx.get(url).text"
    ]
   },
@@ -858,10 +837,13 @@
     "def llms_txt2ctx(\n",
     "    fname:str, # File name to read\n",
     "    optional:bool_arg=False, # Include 'optional' section?\n",
-    "    n_workers:int=None # Number of threads to use for parallel downloading\n",
+    "    n_workers:int=None, # Number of threads to use for parallel downloading\n",
+    "    save_nbdev:bool_arg=False #save output to nbdev `docs_path` instead of emitting to stdout\n",
     "):\n",
     "    \"Print a `Project` with a `Section` for each H2 part in file read from `fname`, optionally skipping the 'optional' section.\"\n",
-    "    print(create_ctx(Path(fname).read_text(), optional=optional, n_workers=n_workers))"
+    "    ctx = create_ctx(Path(fname).read_text(), optional=optional, n_workers=n_workers)\n",
+    "    if save_nbdev: (get_config().doc_path / fname).write_text(ctx)\n",
+    "    else: print(ctx)"
    ]
   },
   {

diff --git a/settings.ini b/settings.ini
@@ -27,7 +27,7 @@ keywords = nbdev jupyter notebook python LLMs NLP
 language = English
 status = 3
 user = AnswerDotAI
-requirements = fastcore>=1.7.3 httpx
+requirements = fastcore>=1.7.3 httpx nbdev
 conda_user = fastai
 console_scripts = llms_txt2ctx=llms_txt.core:llms_txt2ctx
 readme_nb = index.ipynb