init commit

.github/workflows/deploy.yaml

@@ -0,0 +1,14 @@
+name: Deploy to GitHub Pages
+
+permissions:
+  contents: write
+  pages: write
+
+on:
+  push:
+    branches: [ "main", "master" ]
+  workflow_dispatch:
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps: [uses: fastai/workflows/quarto-ghp@master]

.github/workflows/test.yaml.off

@@ -0,0 +1,7 @@
+name: CI
+on:  [workflow_dispatch, pull_request, push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps: [uses: fastai/workflows/nbdev-ci@master]

.gitignore

@@ -1,3 +1,26 @@
+fasthtml.md
+posts/
+.quarto
+.sesskey
+*.db-*
+*.db
+.gitattributes
+_proc/
+sidebar.yml
+Gemfile.lock
+token
+_docs/
+conda/
+.last_checked
+.gitconfig
+*.bak
+*.log
+*~
+~*
+_tmp*
+tmp*
+tags
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -8,6 +31,7 @@ __pycache__/
 
 # Distribution / packaging
 .Python
+env/
 build/
 develop-eggs/
 dist/
@@ -20,11 +44,9 @@ parts/
 sdist/
 var/
 wheels/
-share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
-MANIFEST
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -39,17 +61,13 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
-.nox/
 .coverage
 .coverage.*
 .cache
 nosetests.xml
 coverage.xml
 *.cover
-*.py,cover
 .hypothesis/
-.pytest_cache/
-cover/
 
 # Translations
 *.mo
@@ -58,8 +76,6 @@ cover/
 # Django stuff:
 *.log
 local_settings.py
-db.sqlite3
-db.sqlite3-journal
 
 # Flask stuff:
 instance/
@@ -72,63 +88,27 @@ instance/
 docs/_build/
 
 # PyBuilder
-.pybuilder/
 target/
 
 # Jupyter Notebook
 .ipynb_checkpoints
 
-# IPython
-profile_default/
-ipython_config.py
-
 # pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
-.pdm.toml
-.pdm-python
-.pdm-build/
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
+.python-version
+
+# celery beat schedule file
 celerybeat-schedule
-celerybeat.pid
 
 # SageMath parsed files
 *.sage.py
 
-# Environments
+# dotenv
 .env
+
+# virtualenv
 .venv
-env/
 venv/
 ENV/
-env.bak/
-venv.bak/
 
 # Spyder project settings
 .spyderproject
@@ -142,21 +122,32 @@ venv.bak/
 
 # mypy
 .mypy_cache/
-.dmypy.json
-dmypy.json
 
-# Pyre type checker
-.pyre/
+.vscode
+*.swp
+
+# osx generated files
+.DS_Store
+.DS_Store?
+.Trashes
+ehthumbs.db
+Thumbs.db
+.idea
+
+# pytest
+.pytest_cache
+
+# tools/trust-doc-nbs
+docs_src/.last_checked
+
+# symlinks to fastai
+docs_src/fastai
+tools/fastai
 
-# pytype static type analyzer
-.pytype/
+# link checker
+checklink/cookies.txt
 
-# Cython debug symbols
-cython_debug/
+# .gitconfig is now autogenerated
+.gitconfig
 
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+_docs

MANIFEST.in

@@ -0,0 +1,5 @@
+include settings.ini
+include LICENSE
+include CONTRIBUTING.md
+include README.md
+recursive-exclude * __pycache__

README.md

@@ -1,2 +1,106 @@
 # llms-txt
-The /llms.txt file, helping language models use your website
+
+
+<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->
+
+This file will become your README and also the index of your
+documentation.
+
+## Developer Guide
+
+If you are new to using `nbdev` here are some useful pointers to get you
+started.
+
+### Setup
+
+It can be helpful to have a dedicated environment for development. Here
+we are assuming that you have an conda environment file called `env.yml`
+named after `llms_txt` i.e.:
+
+``` yaml
+# env.yml
+name: llms_txt
+
+channels:
+  - fastai
+
+dependencies:
+  - fastai::nbdev>=2.3.12
+# - python>=3.11  # specify python version if required 
+# - dependency 1
+# - dependency 2
+# - pip
+# pip:
+#   - pip dependency 1
+#   - pip dependency 2
+```
+
+You can then use `conda` or `mamba` (faster at resolving) to create and
+update your environment file should your needs change as you work on
+`llms_txt`
+
+``` sh
+# create a conda environment for working on llms-txt
+$ mamba env create -f env.yml
+
+# update conda environment
+$ mamba env update -n llms_txt --file env.yml
+```
+
+### Install llms_txt in Development mode
+
+``` sh
+# activate conda environment
+$ conda activate llms_txt
+
+# make sure llms_txt package is installed in development mode
+$ pip install -e .
+
+# make changes under nbs/ directory
+# ...
+
+# compile to have changes apply to llms_txt
+$ nbdev_prepare
+```
+
+## Usage
+
+### Installation
+
+Install latest from the GitHub
+[repository](https://github.com/AnswerDotAI/llms-txt):
+
+``` sh
+$ pip install git+https://github.com/AnswerDotAI/llms-txt.git
+```
+
+or from [conda](https://anaconda.org/AnswerDotAI/llms-txt)
+
+``` sh
+$ conda install -c AnswerDotAI llms_txt
+```
+
+or from [pypi](https://pypi.org/project/llms-txt/)
+
+``` sh
+$ pip install llms_txt
+```
+
+### Documentation
+
+Documentation can be found hosted on this GitHub
+[repository](https://github.com/AnswerDotAI/llms-txt)’s
+[pages](https://AnswerDotAI.github.io/llms-txt/). Additionally you can
+find package manager specific guidelines on
+[conda](https://anaconda.org/AnswerDotAI/llms-txt) and
+[pypi](https://pypi.org/project/llms-txt/) respectively.
+
+## How to use
+
+Fill me in please! Don’t forget code examples:
+
+``` python
+1+1
+```
+
+    2

llms_txt/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

llms_txt/_modidx.py

@@ -0,0 +1,17 @@
+# Autogenerated by nbdev
+
+d = { 'settings': { 'branch': 'main',
+                'doc_baseurl': '/llms-txt',
+                'doc_host': 'https://AnswerDotAI.github.io',
+                'git_url': 'https://github.com/AnswerDotAI/llms-txt',
+                'lib_path': 'llms_txt'},
+  'syms': { 'llms_txt.core': { 'llms_txt.core.Doc': ('core.html#doc', 'llms_txt/core.py'),
+                               'llms_txt.core.Section': ('core.html#section', 'llms_txt/core.py'),
+                               'llms_txt.core._opt_re': ('core.html#_opt_re', 'llms_txt/core.py'),
+                               'llms_txt.core._parse_llms_txt': ('core.html#_parse_llms_txt', 'llms_txt/core.py'),
+                               'llms_txt.core._parse_section': ('core.html#_parse_section', 'llms_txt/core.py'),
+                               'llms_txt.core._split_on_h2': ('core.html#_split_on_h2', 'llms_txt/core.py'),
+                               'llms_txt.core.get_sizes': ('core.html#get_sizes', 'llms_txt/core.py'),
+                               'llms_txt.core.llms_txt2ctx': ('core.html#llms_txt2ctx', 'llms_txt/core.py'),
+                               'llms_txt.core.mk_ctx': ('core.html#mk_ctx', 'llms_txt/core.py'),
+                               'llms_txt.core.parse_llms_file': ('core.html#parse_llms_file', 'llms_txt/core.py')}}}