paraffin v0.3.0

.github/workflows/publish.yaml

@@ -14,6 +14,10 @@ jobs:
           python-version: "3.12"
       - name: Install Poetry
         uses: snok/install-poetry@v1
+      - name: build frontend
+        run: |
+          npm install -g bun
+          cd app && bun install && bun vite build && cd ..
       - name: Publish
         env:
           PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/tests.yaml

@@ -34,6 +34,10 @@ jobs:
       - name: Install package
         run: |
           poetry install --no-interaction
+      - name: build frontend
+        run: |
+          npm install -g bun
+          cd app && bun install && bun vite build && cd ..
       - name: Setup git user
         run: |
           git config --global user.name "John Doe"

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: check-case-conflict
       - id: check-docstring-first
       - id: check-executables-have-shebangs
-      - id: check-json
+      # - id: check-json
       - id: check-merge-conflict
         args: ['--assume-in-merge']
       - id: check-toml

README.md

@@ -2,12 +2,6 @@
 [![PyPI version](https://badge.fury.io/py/paraffin.svg)](https://badge.fury.io/py/paraffin)
 [![Discord](https://img.shields.io/discord/1034511611802689557)](https://discord.gg/7ncfwhsnm4)
 
-> [!WARNING]
-> [celery](https://github.com/celery/celery) can not handle large workflows and will crash your computer.
-> See https://github.com/celery/celery/issues/9475.
-> Therefore, `paraffin` should **currently not** be used for large workflows.
-> We are working on a solution without relying on `celery`.
-
 # paraffin
 
 Paraffin, derived from the Latin phrase `parum affinis` meaning
@@ -31,55 +25,30 @@ pip install paraffin
 
 ## Usage
 
-The `paraffin submit` command mirrors `dvc repro`, enabling you to queue and execute your entire pipeline or selected stages with parallelization.
-If no parameters are specified, the entire graph will be queued and executed via `dvc repro --single-item`.
+### paraffin submit
+You can submit your current DVC workflow to a database file `paraffin.db` for later execution.
 
+> [!TIP]
+> The paraffin submit command supports globing patterns.
 ```bash
-paraffin submit <stage name> <stage name> ... <stage name>
-# Example: run with a maximum of 4 parallel jobs
-paraffin worker --concurrency=4
-```
-
-### Parallel Execution
-Due to limitations in Celery’s graph handling (see [Celery discussion](https://github.com/celery/celery/discussions/9376)), complete parallelization is not always achievable. Paraffin will display parallel-ready stages in a flowchart format.
-All stages are visualized in a [Mermaid](https://mermaid.js.org/) flowchart.
-
-```mermaid
-flowchart TD
-        subgraph Level0:1
-                A_X_ParamsToOuts
-                A_X_ParamsToOuts_1
-                A_Y_ParamsToOuts
-                A_Y_ParamsToOuts_1
-        end
-        subgraph Level0:2
-                A_X_AddNodeNumbers
-                A_Y_AddNodeNumbers
-        end
-        subgraph Level0:3
-                A_SumNodeAttributes
-        end
-        Level0:1 --> Level0:2
-        Level0:2 --> Level0:3
-        subgraph Level1:1
-                B_X_ParamsToOuts
-                B_X_ParamsToOuts_1
-                B_Y_ParamsToOuts
-                B_Y_ParamsToOuts_1
-        end
-        subgraph Level1:2
-                B_X_AddNodeNumbers
-                B_Y_AddNodeNumbers
-        end
-        subgraph Level1:3
-                B_SumNodeAttributes
-        end
-        Level1:1 --> Level1:2
-        Level1:2 --> Level1:3
+paraffin submit C_AddNodeNumbers "A*"
 ```
 
+### paraffin worker
+A submitted job will be executed by paraffin workers.
+To start a worker you can run `paraffin worker`.
+The worker will pick up all the jobs in the workeres queue and close once finished.
 
+```bash
+paraffin worker
+```
 
+### paraffin ui
+Paraffin ships with a web application for visualizing the progress.
+You can start it using
+```bash
+paraffin ui
+```
 
 ## Queue Labels
 
@@ -93,9 +62,9 @@ queue:
 ```
 Then, start a worker with specified queues, such as celery (default) and AQueue:
 ```bash
-paraffin worker -q AQueue,celery
+paraffin worker -q AQueue,default
 ```
-All `stages` not assigned to a queue in `paraffin.yaml` will default to the `celery` queue.
+All `stages` not assigned to a queue in `paraffin.yaml` will default to the `default` queue.
 
 
 > [!TIP]

app/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

app/README.md

@@ -0,0 +1,50 @@
+# React + TypeScript + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
+
+## Expanding the ESLint configuration
+
+If you are developing a production application, we recommend updating the configuration to enable type aware lint rules:
+
+- Configure the top-level `parserOptions` property like this:
+
+```js
+export default tseslint.config({
+  languageOptions: {
+    // other options...
+    parserOptions: {
+      project: ['./tsconfig.node.json', './tsconfig.app.json'],
+      tsconfigRootDir: import.meta.dirname,
+    },
+  },
+})
+```
+
+- Replace `tseslint.configs.recommended` to `tseslint.configs.recommendedTypeChecked` or `tseslint.configs.strictTypeChecked`
+- Optionally add `...tseslint.configs.stylisticTypeChecked`
+- Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and update the config:
+
+```js
+// eslint.config.js
+import react from 'eslint-plugin-react'
+
+export default tseslint.config({
+  // Set the react version
+  settings: { react: { version: '18.3' } },
+  plugins: {
+    // Add the react plugin
+    react,
+  },
+  rules: {
+    // other rules...
+    // Enable its recommended rules
+    ...react.configs.recommended.rules,
+    ...react.configs['jsx-runtime'].rules,
+  },
+})
+```

app/biome.json

@@ -0,0 +1,30 @@
+{
+	"$schema": "https://biomejs.dev/schemas/1.9.4/schema.json",
+	"vcs": {
+		"enabled": false,
+		"clientKind": "git",
+		"useIgnoreFile": false
+	},
+	"files": {
+		"ignoreUnknown": false,
+		"ignore": []
+	},
+	"formatter": {
+		"enabled": true,
+		"indentStyle": "tab"
+	},
+	"organizeImports": {
+		"enabled": true
+	},
+	"linter": {
+		"enabled": true,
+		"rules": {
+			"recommended": true
+		}
+	},
+	"javascript": {
+		"formatter": {
+			"quoteStyle": "double"
+		}
+	}
+}

app/eslint.config.js

@@ -0,0 +1,28 @@
+import js from "@eslint/js";
+import globals from "globals";
+import reactHooks from "eslint-plugin-react-hooks";
+import reactRefresh from "eslint-plugin-react-refresh";
+import tseslint from "typescript-eslint";
+
+export default tseslint.config(
+	{ ignores: ["dist"] },
+	{
+		extends: [js.configs.recommended, ...tseslint.configs.recommended],
+		files: ["**/*.{ts,tsx}"],
+		languageOptions: {
+			ecmaVersion: 2020,
+			globals: globals.browser,
+		},
+		plugins: {
+			"react-hooks": reactHooks,
+			"react-refresh": reactRefresh,
+		},
+		rules: {
+			...reactHooks.configs.recommended.rules,
+			"react-refresh/only-export-components": [
+				"warn",
+				{ allowConstantExport: true },
+			],
+		},
+	},
+);

app/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + React + TS</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

app/package.json

@@ -0,0 +1,36 @@
+{
+	"name": "app",
+	"private": true,
+	"version": "0.0.0",
+	"type": "module",
+	"scripts": {
+		"dev": "vite",
+		"build": "tsc -b && vite build",
+		"lint": "eslint .",
+		"preview": "vite preview"
+	},
+	"dependencies": {
+		"@xyflow/react": "^12.4.2",
+		"bootstrap": "^5.3.3",
+		"elkjs": "^0.9.3",
+		"react": "^18.3.1",
+		"react-bootstrap": "^2.10.8",
+		"react-dom": "^18.3.1",
+		"react-icons": "^5.4.0",
+		"react-markdown": "^9.0.3"
+	},
+	"devDependencies": {
+		"@biomejs/biome": "1.9.4",
+		"@eslint/js": "^9.17.0",
+		"@types/react": "^18.3.18",
+		"@types/react-dom": "^18.3.5",
+		"@vitejs/plugin-react": "^4.3.4",
+		"eslint": "^9.17.0",
+		"eslint-plugin-react-hooks": "^5.0.0",
+		"eslint-plugin-react-refresh": "^0.4.16",
+		"globals": "^15.14.0",
+		"typescript": "~5.6.2",
+		"typescript-eslint": "^8.18.2",
+		"vite": "^6.0.5"
+	}
+}

app/src/App.css

@@ -0,0 +1,10 @@
+#root {
+	max-width: 1280px;
+	margin: 0 auto;
+	padding: 2rem;
+	text-align: center;
+}
+
+.react-flow__node {
+	z-index: -1 !important;
+}