Skip to content

Commit

Permalink
Change folder structure to group python, javascript and css, add in a…
Browse files Browse the repository at this point in the history
…bility to create data contracts, fix export to HTML, RDF and Avro IDL
  • Loading branch information
pflooky committed Aug 13, 2024
1 parent 35e6f17 commit bb311ac
Show file tree
Hide file tree
Showing 26 changed files with 13,948 additions and 287 deletions.
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,18 @@ Playground site for creating/validating data contracts

[Try here.](https://data-catering.github.io/data-contract-playground/)

## Create

Given input data from a data source, create a data contract from it.
Uses [datacontract-cli](https://github.com/datacontract/datacontract-cli) to create data contracts.
Current input formats supported:
- Avro
- BigQuery
- DBT
- JSON Schema
- SQL
- Unity Catalog

## Export

Given a data contract, export it to different formats.
Expand Down
136 changes: 136 additions & 0 deletions create.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
<!doctype html>

<html>
<head>
<!-- Google tag (gtag.js) -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-S3ZRGKSGBF"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());

gtag('config', 'G-S3ZRGKSGBF');
</script>
<title>Create - Data Contract Playground</title>

<!-- Recommended meta tags -->
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width,initial-scale=1.0">

<!-- PyScript CSS -->
<link rel="stylesheet" href="https://pyscript.net/releases/2024.1.1/core.css">

<!-- This script tag bootstraps PyScript -->
<script type="module" src="https://pyscript.net/releases/2024.1.1/core.js"></script>
<script src="https://www.unpkg.com/ace-builds@latest/src-noconflict/ace.js"></script>
<script src="https://www.unpkg.com/ace-builds@latest/src-noconflict/ext-language_tools.js"></script>
<script src="https://www.unpkg.com/ace-builds@latest/src-noconflict/theme-chrome.js"></script>
<script src="https://cdn.jsdelivr.net/npm/ace-linters"></script>

<script type="module">
const loading = document.getElementById('loading');
addEventListener('py:ready', () => loading.close());
loading.showModal();
</script>

<link rel="icon" href="image/data_catering_transparent.svg"/>
<link rel="stylesheet" href="src/css/main.css" type="text/css"/>
<link rel="stylesheet" href="src/css/pyscript.css" type="text/css"/>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">

<meta property="description" content="Create data contracts from different formats. Formats include Avro, BigQuery, DBT, JSON Schema, SQL and Unity Catalog.">
<meta property="og:type" content="website">
<meta property="og:title" content="Create - Data Contract Playground">
<meta property="og:description" content="Create data contracts from different formats. Formats include Avro, BigQuery, DBT, JSON Schema, SQL and Unity Catalog.">
<meta property="og:image" content="image/social/data-contract-playground-create.png">
<meta property="og:image:type" content="image/png">
<meta property="og:image:width" content="1200">
<meta property="og:image:height" content="630">
<meta property="twitter:card" content="summary_large_image">
<meta property="twitter:title" content="Create - Data Contract Playground">
<meta property="twitter:description" content="Create data contracts from different formats. Formats include Avro, BigQuery, DBT, JSON Schema, SQL and Unity Catalog.">
<meta property="twitter:image" content="image/social/data-contract-playground-create.png">
</head>

<body>
<dialog id="loading">
<h1>Loading...</h1>
</dialog>

<div class="top-banner">
<span>
<div class="logo-title">
<a href="https://data.catering/"><img class="logo" src="image/data_catering_transparent.svg" alt="logo"/></a>
<b>Data Contract Playground</b>
</div>
</span>
<a class="github-button" href="https://github.com/data-catering/data-contract-playground"
data-color-scheme="no-preference: light; light: light; dark: dark;" data-icon="octicon-star" data-size="large"
data-show-count="true" aria-label="Star data-catering/data-contract-playground on GitHub">Star</a>
</div>
<nav class="topnav">
<a href="create.html">Create</a>
<a href="export.html">Export</a>
<a href="index.html">Validate</a>
</nav>

<div>
<section>
<main>
<div class="convert-container">
<div class="convert-text">
<p>Powered by <a href="https://github.com/datacontract/datacontract-cli" target="_blank">datacontract-cli</a></p>
</div>
<div class="convert-button-container">
<button id="convert-btn" type="button" py-click="import_to_output_type">
Convert
</button>
</div>
</div>
<div class="code-containers">
<div class="code-container code-container-input" id="convert-input-container">
<div class="code-container-options">
<label for="convert-input-type">Choose input type:</label>

<select name="convert-input-type" id="convert-input-type"></select>
</div>
<div class="code" id="input-yaml"></div>
</div>
<div class="code-container code-container-output" id="convert-output-container">
<div class="code-container-options">
<label for="convert-output-type">Choose output type:</label>

<select name="convert-output-type" id="convert-output-type"></select>
</div>
<div class="code" id="output-text"></div>
</div>
</div>
</main>
<script type="py" src="src/python/create.py" config="./pyscript.toml"></script>
</section>
</div>

<footer>
<div class="footer-container">
<a class="link footer-left" href="index.html">Data Contract Playground</a>
<div class="footer-icons">
<ul>
<li><a href="https://github.com/data-catering/data-contract-playground" target="_blank"><i
class="fa fa-github fa-2x"></i></a></li>
<li><a href="https://www.linkedin.com/in/peter-flook/" target="_blank"><i
class="fa fa-linkedin fa-2x"></i></a></li>
<li><a href="https://medium.com/@pflooky" target="_blank"><i class="fa fa-medium fa-2x"></i></a></li>
</ul>
</div>
<a class="link footer-right" href="https://data.catering/about/#contact" target="_blank">contact</a>
</div>
</footer>
<script>
function createObject(x, variableName){
eval(variableName + " = x")
}
</script>
<script type="module" src="src/js/create.js"></script>
<script async defer src="https://buttons.github.io/buttons.js"></script>
</body>
</html>
2 changes: 1 addition & 1 deletion datacontract-cli
Submodule datacontract-cli updated 78 files
+21 −5 CHANGELOG.md
+35 −2 README.md
+11 −0 datacontract/engines/fastjsonschema/check_jsonschema.py
+1 −1 datacontract/engines/soda/check_soda_execute.py
+23 −24 datacontract/engines/soda/connections/duckdb.py
+1 −1 datacontract/imports/bigquery_importer.py
+1 −1 datacontract/imports/dbt_importer.py
+62 −58 datacontract/imports/glue_importer.py
+1 −1 datacontract/imports/importer.py
+1 −1 datacontract/imports/odcs_importer.py
+1 −1 datacontract/imports/spark_importer.py
+1 −1 datacontract/imports/sql_importer.py
+1 −1 datacontract/imports/unity_importer.py
+21 −22 pyproject.toml
+1 −0 tests/fixtures/gcs-json-remote/data/README.md
+1,000 −0 tests/fixtures/gcs-json-remote/data/inventory/year=2022/month=04/day=20/hour=00/inventory+0+0001327496.json
+1,000 −0 tests/fixtures/gcs-json-remote/data/inventory/year=2022/month=04/day=20/hour=00/inventory+0+0001328496.json
+1,000 −0 tests/fixtures/gcs-json-remote/data/inventory/year=2022/month=04/day=20/hour=00/inventory+0+0001329496.json
+1,000 −0 tests/fixtures/gcs-json-remote/data/inventory/year=2022/month=04/day=20/hour=00/inventory+0+0001330496.json
+1,000 −0 tests/fixtures/gcs-json-remote/data/inventory/year=2022/month=05/day=04/hour=00/inventory+0+0002657902.json
+1,000 −0 tests/fixtures/gcs-json-remote/data/inventory/year=2022/month=05/day=04/hour=00/inventory+0+0002658902.json
+1,000 −0 tests/fixtures/gcs-json-remote/data/inventory/year=2022/month=05/day=04/hour=00/inventory+0+0002659902.json
+32 −0 tests/fixtures/gcs-json-remote/datacontract.yaml
+25 −4 tests/fixtures/glue/datacontract.yaml
+1 −3 tests/test_breaking.py
+1 −2 tests/test_catalog.py
+1 −3 tests/test_changelog.py
+1 −3 tests/test_cli.py
+1 −2 tests/test_export_avro.py
+1 −3 tests/test_export_bigquery.py
+1 −3 tests/test_export_complex_data_contract.py
+1 −2 tests/test_export_custom_exporter.py
+1 −2 tests/test_export_dbml.py
+1 −2 tests/test_export_dbt_models.py
+1 −3 tests/test_export_dbt_sources.py
+1 −3 tests/test_export_dbt_staging_sql.py
+1 −3 tests/test_export_go.py
+1 −2 tests/test_export_great_expectations.py
+1 −2 tests/test_export_html.py
+1 −2 tests/test_export_jsonschema.py
+1 −2 tests/test_export_odcs.py
+1 −3 tests/test_export_protobuf.py
+1 −2 tests/test_export_rdf.py
+1 −3 tests/test_export_spark.py
+1 −3 tests/test_export_sql.py
+1 −3 tests/test_export_sql_query.py
+1 −3 tests/test_export_terraform.py
+1 −3 tests/test_import_avro.py
+1 −3 tests/test_import_bigquery.py
+1 −3 tests/test_import_dbt.py
+55 −5 tests/test_import_glue.py
+1 −3 tests/test_import_jsonschema.py
+1 −2 tests/test_import_odcs.py
+1 −3 tests/test_import_unity_file.py
+1 −3 tests/test_integration_datameshmanager.py
+1 −2 tests/test_integration_opentelemetry.py
+1 −3 tests/test_lint.py
+1 −2 tests/test_resolve.py
+1 −2 tests/test_roundtrip_jsonschema.py
+1 −2 tests/test_test_azure_parquet_remote.py
+1 −2 tests/test_test_bigquery.py
+1 −2 tests/test_test_databricks.py
+1 −3 tests/test_test_examples_csv.py
+1 −3 tests/test_test_examples_formats_valid.py
+1 −3 tests/test_test_examples_inline.py
+1 −3 tests/test_test_examples_json.py
+1 −3 tests/test_test_examples_missing.py
+48 −0 tests/test_test_gcs_json_remote.py
+1 −2 tests/test_test_kafka.py
+1 −2 tests/test_test_kafka_remote.py
+1 −3 tests/test_test_postgres.py
+1 −2 tests/test_test_s3_csv.py
+10 −2 tests/test_test_s3_delta.py
+1 −2 tests/test_test_s3_json_multiple_models.py
+1 −2 tests/test_test_s3_json_remote.py
+1 −3 tests/test_test_snowflake.py
+3 −5 tests/test_test_sqlserver.py
+1 −1 tests/test_test_trino.py
94 changes: 94 additions & 0 deletions example/avro/orders.avsc
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
{
"fields": [
{
"name": "ordertime",
"doc": "My Field",
"type": "long"
},
{
"name": "orderid",
"type": "int"
},
{
"name": "itemid",
"type": "string"
},
{
"name": "material",
"doc": "An optional field",
"type": [
"null",
{
"type": "string",
"avro.java.string": "String"
}
]
},
{
"name": "orderunits",
"type": "double"
},
{
"name": "emailaddresses",
"doc": "Different email addresses of a customer",
"type": {
"type": "array",
"items": "string",
"format": "email",
"pattern": "^.*@.*$"
}
},
{
"name": "address",
"type": {
"fields": [
{
"name": "city",
"type": "string"
},
{
"name": "state",
"type": "string"
},
{
"name": "zipcode",
"type": "long"
}
],
"name": "address",
"type": "record"
}
},
{
"name": "status",
"doc": "order status",
"type": {
"type": "enum",
"name": "Status",
"symbols": ["PLACED", "SHIPPED", "DELIVERED", "CANCELLED"]
}
},
{
"name": "metadata",
"doc": "Additional metadata about the order",
"type": {
"type": "map",
"values": {
"type": "record",
"name": "MetadataValue",
"fields": [
{"name": "value", "type": "string"},
{"name": "type", "type": {"type": "enum", "name": "MetadataType", "symbols": ["STRING", "LONG", "DOUBLE"]}},
{"name": "timestamp", "type": "long"},
{"name": "source", "type": "string"}
]
}
}
}

],
"name": "orders",
"doc": "My Model",
"type": "record",
"namespace": "com.sample.schema"
}
Loading

0 comments on commit bb311ac

Please sign in to comment.