Updating docs for praisonai train

johngoad · Aug 6, 2024 · 3e0e2c7 · 3e0e2c7
1 parent 1bb657e
commit 3e0e2c7
Show file tree

Hide file tree

Showing 14 changed files with 920 additions and 224 deletions.
diff --git a/README.md b/README.md
@@ -36,6 +36,11 @@ Praison AI, leveraging both AutoGen and CrewAI or any other agent framework, rep
 | **Chat** | Chat with 100+ LLMs, single AI Agent | [https://docs.praison.ai/ui/chat](https://docs.praison.ai/ui/chat) |
 | **Code** | Chat with entire Codebase, single AI Agent | [https://docs.praison.ai/ui/code](https://docs.praison.ai/ui/code) |
 
+| Other Features | Description | Docs |
+|---|---|---|
+| **Train** | Fine-tune LLMs using your custom data | [https://docs.praison.ai/train](https://docs.praison.ai/train) |
+
+
 ## Google Colab Multi Agents
 
 |               | Cookbook        | Open in Colab                                                                                                                                                                                                                                  |
@@ -45,9 +50,12 @@ Praison AI, leveraging both AutoGen and CrewAI or any other agent framework, rep
 
 ## Install
 
-| PraisonAI | PraisonAI Code | PraisonAI Chat |
-| --- | --- | --- |
-| `pip install praisonai` | `pip install "praisonai[code]"` | `pip install "praisonai[chat]"` |
+|  | Installation |
+|---|---|
+| **PraisonAI** | `pip install praisonai` |
+| **PraisonAI Code** | `pip install "praisonai[code]"` |
+| **PraisonAI Chat** | `pip install "praisonai[chat]"` |
+| **PraisonAI Train** | `pip install "praisonai[train]"` |
 
 ## Key Features
 

diff --git a/docs/api/praisonai/cli.html b/docs/api/praisonai/cli.html
@@ -31,6 +31,22 @@ <h1 class="title">Module <code>praisonai.cli</code></h1>
 <section>
 </section>
 <section>
+<h2 class="section-title" id="header-functions">Functions</h2>
+<dl>
+<dt id="praisonai.cli.stream_subprocess"><code class="name flex">
+<span>def <span class="ident">stream_subprocess</span></span>(<span>command, env=None)</span>
+</code></dt>
+<dd>
+<div class="desc"><p>Execute a subprocess command and stream the output to the terminal in real-time.</p>
+<h2 id="args">Args</h2>
+<dl>
+<dt><strong><code>command</code></strong> :&ensp;<code>list</code></dt>
+<dd>A list containing the command and its arguments.</dd>
+<dt><strong><code>env</code></strong> :&ensp;<code>dict</code>, optional</dt>
+<dd>Environment variables for the subprocess.</dd>
+</dl></div>
+</dd>
+</dl>
 </section>
 <section>
 <h2 class="section-title" id="header-classes">Classes</h2>
@@ -136,6 +152,57 @@ <h2 id="attributes">Attributes</h2>
             self.create_code_interface()
             return
 
+        if args.agent_file == &#39;train&#39;:
+            package_root = os.path.dirname(os.path.abspath(__file__))
+            config_yaml_destination = os.path.join(os.getcwd(), &#39;config.yaml&#39;)
+
+            # Create config.yaml only if it doesn&#39;t exist or --model or --dataset is provided
+            if not os.path.exists(config_yaml_destination) or args.model or args.dataset:
+                config = generate_config(
+                    model_name=args.model,
+                    hf_model_name=args.hf,
+                    ollama_model_name=args.ollama,
+                    dataset=[{
+                        &#34;name&#34;: args.dataset
+                    }]
+                )
+                with open(&#39;config.yaml&#39;, &#39;w&#39;) as f:
+                    yaml.dump(config, f, default_flow_style=False, indent=2) 
+
+            # Overwrite huggingface_save and ollama_save if --hf or --ollama are provided 
+            if args.hf:
+                config[&#34;huggingface_save&#34;] = &#34;true&#34;
+            if args.ollama:
+                config[&#34;ollama_save&#34;] = &#34;true&#34;
+
+            if &#39;init&#39; in sys.argv:
+                from praisonai.setup.setup_conda_env import main as setup_conda_main
+                setup_conda_main()
+                print(&#34;All packages installed&#34;)
+                return
+
+            try:
+                result = subprocess.check_output([&#39;conda&#39;, &#39;env&#39;, &#39;list&#39;])
+                if &#39;praison_env&#39; in result.decode(&#39;utf-8&#39;):
+                    print(&#34;Conda environment &#39;praison_env&#39; found.&#34;)
+                else:
+                    raise subprocess.CalledProcessError(1, &#39;grep&#39;)
+            except subprocess.CalledProcessError:
+                print(&#34;Conda environment &#39;praison_env&#39; not found. Setting it up...&#34;)
+                from praisonai.setup.setup_conda_env import main as setup_conda_main
+                setup_conda_main()
+                print(&#34;All packages installed.&#34;)
+
+            train_args = sys.argv[2:]  # Get all arguments after &#39;train&#39;
+            train_script_path = os.path.join(package_root, &#39;train.py&#39;)
+
+            # Set environment variables
+            env = os.environ.copy()
+            env[&#39;PYTHONUNBUFFERED&#39;] = &#39;1&#39;
+
+            stream_subprocess([&#39;conda&#39;, &#39;run&#39;, &#39;--no-capture-output&#39;, &#39;--name&#39;, &#39;praison_env&#39;, &#39;python&#39;, &#39;-u&#39;, train_script_path, &#39;train&#39;], env=env)
+            return
+
         invocation_cmd = &#34;praisonai&#34;
         version_string = f&#34;PraisonAI version {__version__}&#34;
 
@@ -206,7 +273,11 @@ <h2 id="attributes">Attributes</h2>
         parser.add_argument(&#34;--auto&#34;, nargs=argparse.REMAINDER, help=&#34;Enable auto mode and pass arguments for it&#34;)
         parser.add_argument(&#34;--init&#34;, nargs=argparse.REMAINDER, help=&#34;Enable auto mode and pass arguments for it&#34;)
         parser.add_argument(&#34;agent_file&#34;, nargs=&#34;?&#34;, help=&#34;Specify the agent file&#34;)
-        parser.add_argument(&#34;--deploy&#34;, action=&#34;store_true&#34;, help=&#34;Deploy the application&#34;)  # New argument
+        parser.add_argument(&#34;--deploy&#34;, action=&#34;store_true&#34;, help=&#34;Deploy the application&#34;) 
+        parser.add_argument(&#34;--model&#34;, type=str, help=&#34;Model name&#34;)
+        parser.add_argument(&#34;--hf&#34;, type=str, help=&#34;Hugging Face model name&#34;)
+        parser.add_argument(&#34;--ollama&#34;, type=str, help=&#34;Ollama model name&#34;)
+        parser.add_argument(&#34;--dataset&#34;, type=str, help=&#34;Dataset name for training&#34;, default=&#34;yahma/alpaca-cleaned&#34;)
         args, unknown_args = parser.parse_known_args()
 
         if unknown_args and unknown_args[0] == &#39;-b&#39; and unknown_args[1] == &#39;api:app&#39;:
@@ -497,6 +568,11 @@ <h2 id="example">Example</h2>
 <li><code><a title="praisonai" href="index.html">praisonai</a></code></li>
 </ul>
 </li>
+<li><h3><a href="#header-functions">Functions</a></h3>
+<ul class="">
+<li><code><a title="praisonai.cli.stream_subprocess" href="#praisonai.cli.stream_subprocess">stream_subprocess</a></code></li>
+</ul>
+</li>
 <li><h3><a href="#header-classes">Classes</a></h3>
 <ul>
 <li>

diff --git a/docs/api/praisonai/inc/config.html b/docs/api/praisonai/inc/config.html
@@ -0,0 +1,69 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1">
+<meta name="generator" content="pdoc3 0.11.1">
+<title>praisonai.inc.config API documentation</title>
+<meta name="description" content="">
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/13.0.0/sanitize.min.css" integrity="sha512-y1dtMcuvtTMJc1yPgEqF0ZjQbhnc/bFhyvIyVNb9Zk5mIGtqVaAB1Ttl28su8AvFMOY0EwRbAe+HCLqj6W7/KA==" crossorigin>
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/13.0.0/typography.min.css" integrity="sha512-Y1DYSb995BAfxobCkKepB1BqJJTPrOp3zPL74AWFugHHmmdcvO+C48WLrUOlhGMc0QG7AE3f7gmvvcrmX2fDoA==" crossorigin>
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css" crossorigin>
+<style>:root{--highlight-color:#fe9}.flex{display:flex !important}body{line-height:1.5em}#content{padding:20px}#sidebar{padding:1.5em;overflow:hidden}#sidebar > *:last-child{margin-bottom:2cm}.http-server-breadcrumbs{font-size:130%;margin:0 0 15px 0}#footer{font-size:.75em;padding:5px 30px;border-top:1px solid #ddd;text-align:right}#footer p{margin:0 0 0 1em;display:inline-block}#footer p:last-child{margin-right:30px}h1,h2,h3,h4,h5{font-weight:300}h1{font-size:2.5em;line-height:1.1em}h2{font-size:1.75em;margin:2em 0 .50em 0}h3{font-size:1.4em;margin:1.6em 0 .7em 0}h4{margin:0;font-size:105%}h1:target,h2:target,h3:target,h4:target,h5:target,h6:target{background:var(--highlight-color);padding:.2em 0}a{color:#058;text-decoration:none;transition:color .2s ease-in-out}a:visited{color:#503}a:hover{color:#b62}.title code{font-weight:bold}h2[id^="header-"]{margin-top:2em}.ident{color:#900;font-weight:bold}pre code{font-size:.8em;line-height:1.4em;padding:1em;display:block}code{background:#f3f3f3;font-family:"DejaVu Sans Mono",monospace;padding:1px 4px;overflow-wrap:break-word}h1 code{background:transparent}pre{border-top:1px solid #ccc;border-bottom:1px solid #ccc;margin:1em 0}#http-server-module-list{display:flex;flex-flow:column}#http-server-module-list div{display:flex}#http-server-module-list dt{min-width:10%}#http-server-module-list p{margin-top:0}.toc ul,#index{list-style-type:none;margin:0;padding:0}#index code{background:transparent}#index h3{border-bottom:1px solid #ddd}#index ul{padding:0}#index h4{margin-top:.6em;font-weight:bold}@media (min-width:200ex){#index .two-column{column-count:2}}@media (min-width:300ex){#index .two-column{column-count:3}}dl{margin-bottom:2em}dl dl:last-child{margin-bottom:4em}dd{margin:0 0 1em 3em}#header-classes + dl > dd{margin-bottom:3em}dd dd{margin-left:2em}dd p{margin:10px 0}.name{background:#eee;font-size:.85em;padding:5px 10px;display:inline-block;min-width:40%}.name:hover{background:#e0e0e0}dt:target .name{background:var(--highlight-color)}.name > span:first-child{white-space:nowrap}.name.class > span:nth-child(2){margin-left:.4em}.inherited{color:#999;border-left:5px solid #eee;padding-left:1em}.inheritance em{font-style:normal;font-weight:bold}.desc h2{font-weight:400;font-size:1.25em}.desc h3{font-size:1em}.desc dt code{background:inherit}.source summary,.git-link-div{color:#666;text-align:right;font-weight:400;font-size:.8em;text-transform:uppercase}.source summary > *{white-space:nowrap;cursor:pointer}.git-link{color:inherit;margin-left:1em}.source pre{max-height:500px;overflow:auto;margin:0}.source pre code{font-size:12px;overflow:visible}.hlist{list-style:none}.hlist li{display:inline}.hlist li:after{content:',\2002'}.hlist li:last-child:after{content:none}.hlist .hlist{display:inline;padding-left:1em}img{max-width:100%}td{padding:0 .5em}.admonition{padding:.1em 1em;margin-bottom:1em}.admonition-title{font-weight:bold}.admonition.note,.admonition.info,.admonition.important{background:#aef}.admonition.todo,.admonition.versionadded,.admonition.tip,.admonition.hint{background:#dfd}.admonition.warning,.admonition.versionchanged,.admonition.deprecated{background:#fd4}.admonition.error,.admonition.danger,.admonition.caution{background:lightpink}</style>
+<style media="screen and (min-width: 700px)">@media screen and (min-width:700px){#sidebar{width:30%;height:100vh;overflow:auto;position:sticky;top:0}#content{width:70%;max-width:100ch;padding:3em 4em;border-left:1px solid #ddd}pre code{font-size:1em}.name{font-size:1em}main{display:flex;flex-direction:row-reverse;justify-content:flex-end}.toc ul ul,#index ul ul{padding-left:1em}.toc > ul > li{margin-top:.5em}}</style>
+<style media="print">@media print{#sidebar h1{page-break-before:always}.source{display:none}}@media print{*{background:transparent !important;color:#000 !important;box-shadow:none !important;text-shadow:none !important}a[href]:after{content:" (" attr(href) ")";font-size:90%}a[href][title]:after{content:none}abbr[title]:after{content:" (" attr(title) ")"}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,h3{orphans:3;widows:3}h1,h2,h3,h4,h5,h6{page-break-after:avoid}}</style>
+<script defer src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js" integrity="sha512-D9gUyxqja7hBtkWpPWGt9wfbfaMGVt9gnyCvYa+jojwwPHLCzUm5i8rpk7vD7wNee9bA35eYIjobYPaQuKS1MQ==" crossorigin></script>
+<script>window.addEventListener('DOMContentLoaded', () => {
+hljs.configure({languages: ['bash', 'css', 'diff', 'graphql', 'ini', 'javascript', 'json', 'plaintext', 'python', 'python-repl', 'rust', 'shell', 'sql', 'typescript', 'xml', 'yaml']});
+hljs.highlightAll();
+})</script>
+</head>
+<body>
+<main>
+<article id="content">
+<header>
+<h1 class="title">Module <code>praisonai.inc.config</code></h1>
+</header>
+<section id="section-intro">
+</section>
+<section>
+</section>
+<section>
+</section>
+<section>
+<h2 class="section-title" id="header-functions">Functions</h2>
+<dl>
+<dt id="praisonai.inc.config.generate_config"><code class="name flex">
+<span>def <span class="ident">generate_config</span></span>(<span>ollama_save=None, huggingface_save=None, train=None, model_name=None, hf_model_name=None, ollama_model_name=None, model_parameters=None, max_seq_length=None, load_in_4bit=None, lora_r=None, lora_target_modules=None, lora_alpha=None, lora_dropout=None, lora_bias=None, use_gradient_checkpointing=None, random_state=None, use_rslora=None, loftq_config=None, dataset=None, dataset_text_field=None, dataset_num_proc=None, packing=None, per_device_train_batch_size=None, gradient_accumulation_steps=None, warmup_steps=None, num_train_epochs=None, max_steps=None, learning_rate=None, logging_steps=None, optim=None, weight_decay=None, lr_scheduler_type=None, seed=None, output_dir=None, quantization_method=None)</span>
+</code></dt>
+<dd>
+<div class="desc"><p>Generates the configuration for PraisonAI with dynamic overrides.</p></div>
+</dd>
+</dl>
+</section>
+<section>
+</section>
+</article>
+<nav id="sidebar">
+<div class="toc">
+<ul></ul>
+</div>
+<ul id="index">
+<li><h3>Super-module</h3>
+<ul>
+<li><code><a title="praisonai.inc" href="index.html">praisonai.inc</a></code></li>
+</ul>
+</li>
+<li><h3><a href="#header-functions">Functions</a></h3>
+<ul class="">
+<li><code><a title="praisonai.inc.config.generate_config" href="#praisonai.inc.config.generate_config">generate_config</a></code></li>
+</ul>
+</li>
+</ul>
+</nav>
+</main>
+<footer id="footer">
+<p>Generated by <a href="https://pdoc3.github.io/pdoc" title="pdoc: Python API documentation generator"><cite>pdoc</cite> 0.11.1</a>.</p>
+</footer>
+</body>
+</html>
diff --git a/docs/api/praisonai/inc/index.html b/docs/api/praisonai/inc/index.html
@@ -29,6 +29,10 @@ <h1 class="title">Module <code>praisonai.inc</code></h1>
 <section>
 <h2 class="section-title" id="header-submodules">Sub-modules</h2>
 <dl>
+<dt><code class="name"><a title="praisonai.inc.config" href="config.html">praisonai.inc.config</a></code></dt>
+<dd>
+<div class="desc"></div>
+</dd>
 <dt><code class="name"><a title="praisonai.inc.models" href="models.html">praisonai.inc.models</a></code></dt>
 <dd>
 <div class="desc"></div>
@@ -54,6 +58,7 @@ <h2 class="section-title" id="header-submodules">Sub-modules</h2>
 </li>
 <li><h3><a href="#header-submodules">Sub-modules</a></h3>
 <ul>
+<li><code><a title="praisonai.inc.config" href="config.html">praisonai.inc.config</a></code></li>
 <li><code><a title="praisonai.inc.models" href="models.html">praisonai.inc.models</a></code></li>
 </ul>
 </li>

diff --git a/docs/api/praisonai/index.html b/docs/api/praisonai/index.html
@@ -53,6 +53,10 @@ <h2 class="section-title" id="header-submodules">Sub-modules</h2>
 <dd>
 <div class="desc"></div>
 </dd>
+<dt><code class="name"><a title="praisonai.setup" href="setup/index.html">praisonai.setup</a></code></dt>
+<dd>
+<div class="desc"></div>
+</dd>
 <dt><code class="name"><a title="praisonai.test" href="test.html">praisonai.test</a></code></dt>
 <dd>
 <div class="desc"></div>
@@ -87,6 +91,7 @@ <h2 class="section-title" id="header-submodules">Sub-modules</h2>
 <li><code><a title="praisonai.deploy" href="deploy.html">praisonai.deploy</a></code></li>
 <li><code><a title="praisonai.inbuilt_tools" href="inbuilt_tools/index.html">praisonai.inbuilt_tools</a></code></li>
 <li><code><a title="praisonai.inc" href="inc/index.html">praisonai.inc</a></code></li>
+<li><code><a title="praisonai.setup" href="setup/index.html">praisonai.setup</a></code></li>
 <li><code><a title="praisonai.test" href="test.html">praisonai.test</a></code></li>
 <li><code><a title="praisonai.ui" href="ui/index.html">praisonai.ui</a></code></li>
 <li><code><a title="praisonai.version" href="version.html">praisonai.version</a></code></li>