Skip to content

Commit

Permalink
Sandbox (#200)
Browse files Browse the repository at this point in the history
* Docker sandbox

* Working Docker Compose sandbox

* Bootstrap Metabase via entrypoint

* Fmt

* Use sample dbt project from test fixtures
  • Loading branch information
gouline authored Jan 4, 2024
1 parent b6874a7 commit 6846c93
Show file tree
Hide file tree
Showing 18 changed files with 215 additions and 96 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ celerybeat.pid
*.sage.py

# Environments
.env
/.env
.venv
env/
venv/
Expand Down
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,7 @@ dev-install: build
python3 -m pip uninstall -y dbt-metabase \
&& python3 -m pip install dist/dbt_metabase-*-py3-none-any.whl
.PHONY: dev-install

dev-sandbox:
( cd sandbox && docker-compose up --build ; docker-compose down )
.PHONY: dev-sandbox
9 changes: 9 additions & 0 deletions sandbox/.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
POSTGRES_USER=dbtmetabase
POSTGRES_PASSWORD=dbtmetabase1
POSTGRES_DB=dbtmetabase
POSTGRES_SCHEMA=public
POSTGRES_PORT=5432
MB_SETUP_TOKEN=2ffe1bc9-4bc5-4184-a10d-731517d8e4a3
MB_USER=[email protected]
MB_PASSWORD=dbtmetabase1
MB_PORT=3000
1 change: 1 addition & 0 deletions sandbox/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/target/
17 changes: 17 additions & 0 deletions sandbox/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
FROM python:3.11-slim-bullseye

WORKDIR /app

COPY requirements.txt ./

RUN python3 -m pip install -r requirements.txt

COPY entrypoint.py ./
COPY profiles.yml ./

COPY --from=sample_project etc/ ./etc/
COPY --from=sample_project models/ ./models/
COPY --from=sample_project seeds/ ./seeds/
COPY --from=sample_project dbt_project.yml ./

ENTRYPOINT ["python3", "entrypoint.py"]
72 changes: 72 additions & 0 deletions sandbox/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
version: "3"

name: dbt-metabase-sandbox

services:
postgres:
image: postgres:16
environment:
- POSTGRES_USER=${POSTGRES_USER:-}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
- POSTGRES_DB=${POSTGRES_DB:-}
ports:
- ${POSTGRES_PORT}:5432
networks:
- common
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $POSTGRES_USER"]
interval: 5s
timeout: 5s
retries: 5
start_period: 10s
restart: always

metabase:
image: metabase/metabase:latest
environment:
- MB_SETUP_TOKEN=${MB_SETUP_TOKEN:-}
ports:
- ${MB_PORT}:3000
networks:
- common
healthcheck:
test: ["CMD-SHELL", "curl --fail http://localhost:3000/api/health"]
interval: 10s
timeout: 5s
retries: 5
start_period: 20s
restart: always
depends_on:
postgres:
condition: service_healthy

app:
build:
context: .
additional_contexts:
sample_project: ../tests/fixtures/sample_project
command: init
environment:
- POSTGRES_USER=${POSTGRES_USER:-}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
- POSTGRES_DB=${POSTGRES_DB:-}
- POSTGRES_SCHEMA=${POSTGRES_SCHEMA:-}
- POSTGRES_PORT=${POSTGRES_PORT:-}
- POSTGRES_HOST=postgres
- MB_SETUP_TOKEN=${MB_SETUP_TOKEN:-}
- MB_USER=${MB_USER:-}
- MB_PASSWORD=${MB_PASSWORD:-}
- MB_PORT=${MB_PORT:-}
- MB_HOST=metabase
networks:
- common
volumes:
- ./target:/app/target
depends_on:
postgres:
condition: service_healthy
metabase:
condition: service_healthy

networks:
common:
76 changes: 76 additions & 0 deletions sandbox/entrypoint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#!/usr/bin/env python

import requests
from molot import envarg, envarg_int, evaluate, shell, target

POSTGRES_HOST = envarg("POSTGRES_HOST")
POSTGRES_PORT = envarg_int("POSTGRES_PORT")
POSTGRES_DB = envarg("POSTGRES_DB")
POSTGRES_USER = envarg("POSTGRES_USER")
POSTGRES_PASSWORD = envarg("POSTGRES_PASSWORD")
MB_HOST = envarg("MB_HOST")
MB_PORT = envarg_int("MB_PORT")
MB_SETUP_TOKEN = envarg("MB_SETUP_TOKEN")
MB_USER = envarg("MB_USER")
MB_PASSWORD = envarg("MB_PASSWORD")
MB_NAME = envarg("MB_NAME", "dbtmetabase")


@target(description="set up Metabase user and database")
def setup_metabase():
requests.post(
f"http://{MB_HOST}:{MB_PORT}/api/setup",
json={
"token": MB_SETUP_TOKEN,
"user": {
"site_name": MB_NAME,
"first_name": MB_NAME,
"last_name": None,
"email": MB_USER,
"password_confirm": MB_PASSWORD,
"password": MB_PASSWORD,
},
"database": {
"engine": "postgres",
"name": POSTGRES_DB,
"details": {
"host": POSTGRES_HOST,
"port": POSTGRES_PORT,
"dbname": POSTGRES_DB,
"user": POSTGRES_USER,
"password": POSTGRES_PASSWORD,
"schema-filters-type": "all",
"ssl": False,
"tunnel-enabled": False,
"advanced-options": False,
},
"is_on_demand": False,
"is_full_sync": True,
"is_sample": False,
"cache_ttl": None,
"refingerprint": False,
"auto_run_queries": True,
"schedules": {},
},
"prefs": {
"site_name": MB_NAME,
"site_locale": "en",
"allow_tracking": "false",
},
},
timeout=10,
).raise_for_status()


@target(description="run dbt project")
def run_dbt():
shell("dbt seed --profiles-dir .")
shell("dbt run --profiles-dir .")


@target(description="initial setup", depends=["setup_metabase", "run_dbt"])
def init():
pass


evaluate()
11 changes: 11 additions & 0 deletions sandbox/profiles.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
default:
target: default
outputs:
default:
type: postgres
host: "{{ env_var('POSTGRES_HOST') }}"
port: "{{ env_var('POSTGRES_PORT') | int }}"
user: "{{ env_var('POSTGRES_USER') }}"
password: "{{ env_var('POSTGRES_PASSWORD') }}"
dbname: "{{ env_var('POSTGRES_DB') }}"
schema: "{{ env_var('POSTGRES_SCHEMA') }}"
3 changes: 3 additions & 0 deletions sandbox/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
molot
requests
dbt-postgres
2 changes: 0 additions & 2 deletions tests/fixtures/sample_project/.gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,2 @@

dbt_modules/
logs/
**/.DS_Store
79 changes: 2 additions & 77 deletions tests/fixtures/sample_project/README.md
Original file line number Diff line number Diff line change
@@ -1,78 +1,3 @@
## Testing dbt project: `jaffle_shop`
# Sample dbt project

`jaffle_shop` is a fictional ecommerce store. This dbt project transforms raw data from an app database into a customers and orders model ready for analytics.

### What is this repo?
What this repo _is_:
- A self-contained playground dbt project, useful for testing out scripts, and communicating some of the core dbt concepts.

What this repo _is not_:
- A tutorial — check out the [Getting Started Tutorial](https://docs.getdbt.com/tutorial/setting-up) for that. Notably, this repo contains some anti-patterns to make it self-contained, namely the use of seeds instead of sources.
- A demonstration of best practices — check out the [dbt Learn Demo](https://github.com/fishtown-analytics/dbt-learn-demo-v2-archive) repo instead. We want to keep this project as simple as possible. As such, we chose not to implement:
- our standard file naming patterns (which make more sense on larger projects, rather than this five-model project)
- a pull request flow
- CI/CD integrations
- A demonstration of using dbt for a high-complex project, or a demo of advanced features (e.g. macros, packages, hooks, operations) — we're just trying to keep things simple here!

### What's in this repo?
This repo contains [seeds](https://docs.getdbt.com/docs/building-a-dbt-project/seeds) that includes some (fake) raw data from a fictional app.

The raw data consists of customers, orders, and payments, with the following entity-relationship diagram:

![Jaffle Shop ERD](/etc/jaffle_shop_erd.png)


### Running this project
To get up and running with this project:
1. Install dbt using [these instructions](https://docs.getdbt.com/docs/installation).

2. Clone this repository.

3. Change into the `jaffle_shop` directory from the command line:
```bash
$ cd jaffle_shop
```

4. Set up a profile called `jaffle_shop` to connect to a data warehouse by following [these instructions](https://docs.getdbt.com/docs/configure-your-profile). If you have access to a data warehouse, you can use those credentials – we recommend setting your [target schema](https://docs.getdbt.com/docs/configure-your-profile#section-populating-your-profile) to be a new schema (dbt will create the schema for you, as long as you have the right privileges). If you don't have access to an existing data warehouse, you can also setup a local postgres database and connect to it in your profile.

5. Ensure your profile is setup correctly from the command line:
```bash
$ dbt debug
```

6. Load the CSVs with the demo data set. This materializes the CSVs as tables in your target schema. Note that a typical dbt project **does not require this step** since dbt assumes your raw data is already in your warehouse.
```bash
$ dbt seed
```

7. Run the models:
```bash
$ dbt run
```

> **NOTE:** If this steps fails, it might mean that you need to make small changes to the SQL in the models folder to adjust for the flavor of SQL of your target database. Definitely consider this if you are using a community-contributed adapter.
8. Test the output of the models:
```bash
$ dbt test
```

9. Generate documentation for the project:
```bash
$ dbt docs generate
```

10. View the documentation for the project:
```bash
$ dbt docs serve
```

### What is a jaffle?
A jaffle is a toasted sandwich with crimped, sealed edges. Invented in Bondi in 1949, the humble jaffle is an Australian classic. The sealed edges allow jaffle-eaters to enjoy liquid fillings inside the sandwich, which reach temperatures close to the core of the earth during cooking. Often consumed at home after a night out, the most classic filling is tinned spaghetti, while my personal favourite is leftover beef stew with melted cheese.

---
For more information on dbt:
- Read the [introduction to dbt](https://docs.getdbt.com/docs/introduction).
- Read the [dbt viewpoint](https://docs.getdbt.com/docs/about/viewpoint).
- Join the [dbt community](http://community.getdbt.com/).
---
Based on the official [jaffle_shop](https://github.com/dbt-labs/jaffle_shop) sample with several additions and modifications to test certain dbt-metabase features.
Empty file.
24 changes: 10 additions & 14 deletions tests/fixtures/sample_project/dbt_project.yml
Original file line number Diff line number Diff line change
@@ -1,23 +1,19 @@
name: "jaffle_shop"
profile: "default"

name: 'jaffle_shop'
version: '0.1'
profile: 'default'
config-version: 2

source-paths: ["models"]
analysis-paths: ["analysis"]
test-paths: ["tests"]
data-paths: ["data"]
macro-paths: ["macros"]
model-paths: ["models"]
seed-paths: ["seeds"]

target-path: "target"
clean-targets:
- "target"
- "dbt_modules"
- "logs"
- "target"
- "dbt_modules"
- "logs"

models:
jaffle_shop:
materialized: table
staging:
materialized: view
materialized: table
staging:
materialized: view
2 changes: 1 addition & 1 deletion tests/fixtures/sample_project/target/manifest.json

Large diffs are not rendered by default.

9 changes: 8 additions & 1 deletion tests/test_dbt.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import logging
import unittest
from pathlib import Path

from dbtmetabase.dbt import (
DbtReader,
Expand All @@ -14,7 +15,13 @@ class TestDbtReader(unittest.TestCase):
def setUp(self):
"""Must specify dbt root dir"""
self.reader = DbtReader(
manifest_path="tests/fixtures/sample_project/target/manifest.json",
manifest_path=str(
Path("tests")
/ "fixtures"
/ "sample_project"
/ "target"
/ "manifest.json"
),
database="test",
schema="public",
)
Expand Down

0 comments on commit 6846c93

Please sign in to comment.