Resolved conflicts and kept .gitignore from upstream/main

.github/workflows/test.yml

@@ -41,29 +41,29 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@v3
         with:
-          python-version-file: ".python-version"
-          cache: "pip"
-          cache-dependency-path: "**/pyproject.toml"
+          version: ">=0.5.0"
+          enable-cache: true
+          cache-dependency-glob: "**/pyproject.toml"
 
-      - name: Install dependencies
+      - name: Format and lint
         run: |
           python -m pip install --upgrade pip
           pip install hatch pytest pytest-cov
+          uv run ruff check
+          uv run ruff format --check
 
-      - name: Build and lint Widget
-        run: |
-          cd packages/widget
-          hatch build
-          hatch fmt --check
+      - name: Build widget
+        run: uv build --package duckdb-server
 
-      - name: Build, lint, and test Server
+      - name: Build and test server
         run: |
           cd packages/duckdb-server
-          hatch build
-          hatch fmt --check
-          hatch run test:cov
+
+          uv build
+          uv run mypy
+          uv run --with pytest-cov pytest --cov-report=term-missing --color=yes --cov=pkg
 
       - name: Build and test Schema Wrapper
         run: |

.gitignore

@@ -7,20 +7,9 @@ docs/.vitepress/dist
 node_modules
 packages/*/dist
 packages/*/test-output
-transitionFolder
-
-# Virtual environment
-venv/
-.env/
-
-# Conda environment
-conda-env/
-
-# Python cache
-__pycache__/
-*.pyc
-
-# Test cache
-.pytest_cache/
-.coverage
-
+**/__pycache__
+**/.ruff_cache
+**/.pytest_cache
+**/.mypy_cache
+**/.coverage
+**/.venv

.python-version

@@ -1 +1 @@
-3.11
+3.12

README.md

@@ -38,7 +38,7 @@ _Note_: For convenience, the `vgplot` package re-exports much of the `mosaic-cor
 
 ### Core Components
 
-- [`mosaic-core`](https://github.com/uwdata/mosaic/tree/main/packages/core): The core Mosaic components. A central coordinator, parameters and selections for linking scalar values or query predicates (respectively) across Mosaic clients, and filter groups with optimized index management. The Mosaic coordinator can send queries either over the network to a backing server (`socket` and `rest` clients) or to a client-side [DuckDB-WASM](https://github.com/duckdb/duckdb-wasm) instance (`wasm` client).
+- [`mosaic-core`](https://github.com/uwdata/mosaic/tree/main/packages/core): The core Mosaic components. A central coordinator, parameters and selections for linking scalar values or query predicates (respectively) across Mosaic clients, and filter groups with materialized views of pre-aggregated data. The Mosaic coordinator can send queries either over the network to a backing server (`socket` and `rest` clients) or to a client-side [DuckDB-WASM](https://github.com/duckdb/duckdb-wasm) instance (`wasm` client).
 - [`mosaic-sql`](https://github.com/uwdata/mosaic/tree/main/packages/sql): An API for convenient construction and analysis of SQL queries. Query objects then coerce to SQL query strings.
 - [`mosaic-inputs`](https://github.com/uwdata/mosaic/tree/main/packages/inputs): Standalone data-driven components such as input menus, text search boxes, and sortable, load-on-scroll data tables.
 - [`mosaic-plot`](https://github.com/uwdata/mosaic/tree/main/packages/plot): An interactive grammar of graphics implemented on top of [Observable Plot](https://github.com/observablehq/plot). Marks (plot layers) serve as individual Mosaic clients. These marks can push data processing (binning, hex binning, regression) and optimizations (such as M4 for line/area charts) down to the database. This package also provides interactors for linked selection, filtering, and highlighting using Mosaic Params and Selections.
@@ -70,7 +70,7 @@ To run local interactive examples:
 
 To launch a local DuckDB server:
 
-* Install [hatch](https://hatch.pypa.io/latest/install/), if not already present.
+* Install [uv](https://docs.astral.sh/uv/), if not already present.
 * Run `npm run server` to launch the [`duckdb-server`](https://github.com/uwdata/mosaic/tree/main/packages/duckdb-server). This runs the server in development mode, so the server will restart if you change its code.
 
 To use Mosaic with DuckDB Python in Jupyter Notebooks:

dev/bundle.html

@@ -38,7 +38,7 @@
   setConnector();
 
   const queries = [
-    `CREATE TEMP TABLE IF NOT EXISTS flights AS SELECT * FROM read_parquet('data/flights-200k.parquet')`,
+    `CREATE TABLE IF NOT EXISTS flights AS SELECT * FROM read_parquet('data/flights-200k.parquet')`,
     `SELECT count(*) FROM "flights"`
   ]
 

dev/index.html

@@ -23,6 +23,7 @@
         <option value="driving-shifts">Driving Shifts into Reverse</option>
         <option value="earthquakes-feed">Earthquakes Feed</option>
         <option value="earthquakes-globe">Earthquakes Globe</option>
+        <option value="facet-interval">Facet Interval</option>
         <option value="flights-200k">Flights 200k</option>
         <option value="flights-10m">Flights 10M</option>
         <option value="flights-density">Flights Density</option>
@@ -43,6 +44,8 @@
         <option value="pan-zoom">Pan + Zoom</option>
         <option value="population-arrows">Population Arrows</option>
         <option value="presidential-opinion">Presidential Opinion</option>
+        <option value="protein-design">Protein Design</option>
+        <option value="region-tests">Region Tests</option>
         <option value="seattle-temp">Seattle Temperatures</option>
         <option value="sorted-bars">Sorted Bars</option>
         <option value="splom">Scatter Plot Matrix</option>
@@ -84,20 +87,20 @@
           <input id="query-log" type="checkbox" />
         </div>
         <div>
-          Query Cache:
+          Cache Queries:
           <input id="cache" type="checkbox" checked />
         </div>
         <div>
-          Query Consolidation:
+          Consolidate Queries:
           <input id="consolidate" type="checkbox" checked />
         </div>
         <div>
-          Data Cube Indexes:
-          <input id="index" type="checkbox" checked />
+          Pre-aggregate:
+          <input id="preagg" type="checkbox" checked />
         </div>
         <div>
-          Active Index State:
-          <button id="index-state">Log</button>
+          Pre-aggregate State:
+          <button id="preagg-state">Log</button>
         </div>
       </div>
     </details>
@@ -114,32 +117,32 @@
   const qlogToggle = document.querySelector('#query-log');
   const cacheToggle = document.querySelector('#cache');
   const consolidateToggle = document.querySelector('#consolidate');
-  const indexToggle = document.querySelector('#index');
-  const indexState = document.querySelector('#index-state');
+  const preaggToggle = document.querySelector('#preagg');
+  const preaggState = document.querySelector('#preagg-state');
 
   connectorMenu.addEventListener('change', setConnector);
   exampleMenu.addEventListener('change', reload);
   sourceMenu.addEventListener('change', reload);
   qlogToggle.addEventListener('input', setQueryLog);
   cacheToggle.addEventListener('input', setCache);
   consolidateToggle.addEventListener('input', setConsolidate);
-  indexToggle.addEventListener('input', setIndex);
-  indexState.addEventListener('click', () => {
-    const { indexes } = vg.coordinator().dataCubeIndexer || {};
-    if (indexes) {
+  preaggToggle.addEventListener('input', setPreAggregate);
+  preaggState.addEventListener('click', () => {
+    const { entries } = vg.coordinator().preaggregator || {};
+    if (entries) {
       console.warn(
-        'Data Cube Index Entries',
-        Array.from(indexes.values())
+        'Pre-aggregate Entries',
+        Array.from(entries.values())
       );
     } else {
-      console.warn('No Active Data Cube Index');
+      console.warn('No Pre-aggregate Entries');
     }
   });
 
   setQueryLog();
   setCache();
   setConsolidate();
-  setIndex();
+  setPreAggregate();
   setConnector();
 
   async function setConnector() {
@@ -159,8 +162,8 @@
     vg.coordinator().manager.consolidate(consolidateToggle.checked);
   }
 
-  function setIndex() {
-    vg.coordinator().dataCubeIndexer.enabled = indexToggle.checked;
+  function setPreAggregate() {
+    vg.coordinator().preaggregator.enabled = preaggToggle.checked;
   }
 
   function reload() {

dev/notebooks/specs.ipynb

@@ -10,10 +10,11 @@
    "outputs": [],
    "source": [
     "from pprint import pprint\n",
+    "from pathlib import Path\n",
     "\n",
     "import duckdb\n",
     "import ipywidgets as widgets\n",
-    "import yaml\n",
+    "import json\n",
     "\n",
     "from mosaic_widget import MosaicWidget"
    ]
@@ -69,6 +70,22 @@
     "mosaic = MosaicWidget(con=con)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "72656409-7087-4394-bf73-24fe5353cea3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_specs(folder_path):\n",
+    "    folder = Path(folder_path)\n",
+    "    for file_path in sorted(folder.glob(\"*.json\")):\n",
+    "        data = json.loads(file_path.read_text())\n",
+    "        meta = data.get(\"meta\")\n",
+    "        if meta:\n",
+    "            yield (meta.get(\"title\"), file_path)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -78,37 +95,11 @@
    },
    "outputs": [],
    "source": [
+    "weather = Path(\"specs/json/weather.json\")\n",
+    "\n",
     "dropdown = widgets.Dropdown(\n",
-    "    options=[\n",
-    "        (\"Athletes\", \"athletes\"),\n",
-    "        (\"Bias Parameter\", \"bias\"),\n",
-    "        (\"Contours\", \"contours\"),\n",
-    "        (\"Density 1D\", \"density1d\"),\n",
-    "        (\"Density 2D\", \"density2d\"),\n",
-    "        (\"Axes & Gridlines\", \"axes\"),\n",
-    "        (\"Earthquakes\", \"earthquakes\"),\n",
-    "        (\"Flights Density\", \"flights-density\"),\n",
-    "        (\"Flights\", \"flights\"),\n",
-    "        (\"Flights 10M\", \"flights-10m\"),\n",
-    "        (\"Gaia Star Catalog\", \"gaia\"),\n",
-    "        (\"Hexbin\", \"hexbin\"),\n",
-    "        (\"Images\", \"images\"),\n",
-    "        (\"Links\", \"links\"),\n",
-    "        (\"Line Density\", \"line-density\"),\n",
-    "        (\"Mark Types\", \"marks\"),\n",
-    "        (\"Moving Average\", \"moving-average\"),\n",
-    "        (\"Normalize Stocks\", \"normalize\"),\n",
-    "        (\"Overview + Detail\", \"overview-detail\"),\n",
-    "        (\"Pan + Zoom\", \"pan-zoom\"),\n",
-    "        (\"Regression\", \"regression\"),\n",
-    "        (\"Scatter Plot Matrix\", \"splom\"),\n",
-    "        (\"Seattle Weather\", \"weather\"),\n",
-    "        (\"Symbols\", \"symbols\"),\n",
-    "        (\"Table\", \"table\"),\n",
-    "        (\"Voronoi\", \"voronoi\"),\n",
-    "        (\"Wind Map\", \"wind-map\"),\n",
-    "    ],\n",
-    "    value=\"weather\",\n",
+    "    options=get_specs(\"specs/json\"),\n",
+    "    value=weather,\n",
     "    description=\"Example:\",\n",
     ")\n",
     "\n",
@@ -118,13 +109,12 @@
     "\n",
     "\n",
     "def open_spec(spec):\n",
-    "    with open(f\"specs/yaml/{spec}.yaml\") as f:\n",
-    "        mosaic.spec = yaml.safe_load(f)\n",
+    "    mosaic.spec = json.loads(spec.read_text())\n",
     "\n",
     "\n",
     "dropdown.observe(on_change, \"value\")\n",
     "\n",
-    "open_spec(\"weather\")\n",
+    "open_spec(weather)\n",
     "\n",
     "output = widgets.Output()\n",
     "\n",
@@ -148,6 +138,14 @@
    "source": [
     "widgets.VBox([dropdown, mosaic, output])"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "29babf14-24dc-4fc9-befb-ccda6eaa4781",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -166,7 +164,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.8"
+   "version": "3.11.10"
   }
  },
  "nbformat": 4,

dev/notebooks/weather.ipynb

@@ -76,7 +76,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.8"
+   "version": "3.11.10"
   }
  },
  "nbformat": 4,

dev/query/index.html

@@ -59,7 +59,7 @@
     } else {
       await coordinator().exec(
         `DROP TABLE IF EXISTS ${tableName};
-        CREATE TEMP TABLE ${tableName} AS ${input.value}`
+        CREATE TABLE ${tableName} AS ${input.value}`
       );
       coordinator.clear({ catalog: true });
       output.replaceChildren(vg.table({ from: tableName, height: 500 }));

dev/setup.js

@@ -7,6 +7,15 @@ export const vg = createAPIContext();
 // make API accesible for console debugging
 self.vg = vg;
 
+// enable query interface
+self.query = async (sql) => {
+  const r = await vg.coordinator().databaseConnector().query({
+    type: 'arrow',
+    sql
+  });
+  return r.toArray();
+}
+
 export const { coordinator, namedPlots } = vg.context;
 
 export function clear() {

docs/.vitepress/config.js

@@ -133,6 +133,7 @@ export default defineConfig({
             { text: 'Gaia Star Catalog', link: '/examples/gaia' },
             { text: 'Observable Latency', link: '/examples/observable-latency' },
             { text: 'Olympic Athletes', link: '/examples/athletes' },
+            { text: 'Protein Design', link: '/examples/protein-design' },
             { text: 'Pan & Zoom', link: '/examples/pan-zoom' },
             { text: 'Scatter Plot Matrix', link: '/examples/splom' },
             { text: 'Seattle Weather', link: '/examples/weather' }

docs/api/core/client.md

@@ -19,11 +19,11 @@ Create a new client instance. If provided, the [Selection](./selection)-valued _
 Property getter for the Selection that should filter this client.
 The [coordinator](./coordinator) uses this property to provide automatic updates to the client upon selection changes.
 
-## filterIndexable
+## filterStable
 
-`client.filterIndexable`
+`client.filterStable`
 
-Property getter for a Boolean value indicating if the client query can be safely indexed using a pre-aggregated data cube.
+Property getter for a Boolean value indicating if the client query can be safely optimized using a pre-aggregated materialized view.
 This property should return true if changes to the `filterBy` selection do not change the groupby (e.g., binning) values of the client query.
 
 The `MosaicClient` base class will always return `true`.

docs/api/core/coordinator.md

@@ -20,7 +20,7 @@ Create a new Mosaic Coordinator to manage all database communication for clients
 * _logger_: The logger to use, defaults to `console`.
 * _cache_: Boolean flag to enable/disable query caching (default `true`).
 * _consolidate_ Boolean flag to enable/disable query consolidation (default `true`).
-* _indexes_: Data cube indexer options object. The _enabled_ flag (default `true`) determines if data cube indexes should be used when possible. The _schema_ option (default `'mosaic'`) indicates the database schema in which data cube index tables should be created.
+* _preagg_: Pre-aggregation options object. The _enabled_ flag (default `true`) determines if pre-aggregation optimizations should be used when possible. The _schema_ option (default `'mosaic'`) indicates the database schema in which materialized view tables should be created for pre-aggregated data.
 
 ## databaseConnector