Add in ability to export data contracts to other formats

data-catering · Aug 8, 2024 · 6c6f7f4 · 6c6f7f4
1 parent c0b05b7
commit 6c6f7f4
Show file tree

Hide file tree

Showing 14 changed files with 684 additions and 69 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "datacontract-cli"]
+	path = datacontract-cli
+	url = https://github.com/datacontract/datacontract-cli.git
diff --git a/README.md b/README.md
@@ -3,6 +3,30 @@ Playground site for creating/validating data contracts
 
 [Try here.](https://data-catering.github.io/data-contract-playground/)
 
+## Export
+
+Given a data contract, export it to different formats.
+Uses [datacontract-cli](https://github.com/datacontract/datacontract-cli) to export.
+Current formats supported:
+- Avro
+- DBT
+- DBT Sources
+- DBT Staging SQL
+- Go
+- JSON Schema
+- Open Data Contract Standard (ODCS)
+- Protobuf
+- Pydantic Model
+- SodaCL
+- SQL
+- SQL Query
+- Terraform
+
+## Validate
+
+Given a data contract, ensure it adheres to the specification selected.
+[Specifications supported](#support).
+
 ## Support
 
 Currently, it supports:

diff --git a/datacontract-cli b/datacontract-cli
diff --git a/example/data-contract-specification/data-contract-specification.yaml b/example/data-contract-specification/data-contract-specification.yaml
@@ -0,0 +1,53 @@
+dataContractSpecification: 0.9.3
+id: covid_cases
+info:
+  title: COVID-19 cases
+  description: Johns Hopkins University Consolidated data on COVID-19 cases, sourced from Enigma
+  version: "0.0.1"
+  links:
+    blog: https://aws.amazon.com/blogs/big-data/a-public-data-lake-for-analysis-of-covid-19-data/
+    data-explorer: https://dj2taa9i652rf.cloudfront.net/
+    data: https://covid19-lake.s3.us-east-2.amazonaws.com/enigma-jhu/json/part-00000-adec1cd2-96df-4c6b-a5f2-780f092951ba-c000.json
+servers:
+  s3-json:
+    type: s3
+    location: s3://covid19-lake/enigma-jhu/json/*.json
+    format: json
+    delimiter: new_line
+models:
+  covid_cases:
+    description: the number of confirmed covid cases reported for a specified region, with location and county/province/country information.
+    fields:
+      fips:
+        type: string
+        description: state and county two digits code
+      admin2:
+        type: string
+        description: county name
+      province_state:
+        type: string
+        description: province name or state name
+      country_region:
+        type: string
+        description: country name or region name
+      last_update:
+        type: timestamp_ntz
+        description: last update timestamp
+      latitude:
+        type: double
+        description: location (latitude)
+      longitude:
+        type: double
+        description: location (longitude)
+      confirmed:
+        type: int
+        description: number of confirmed cases
+      combined_key:
+        type: string
+        description: county name+state name+country name
+quality:
+  type: SodaCL
+  specification:
+    checks for covid_cases:
+      - freshness(last_update::datetime) < 5000d   # dataset is not updated anymore
+      - row_count > 1000
diff --git a/export.html b/export.html
@@ -0,0 +1,108 @@
+<!doctype html>
+
+<html>
+<head>
+    <title>Data Contract Playground</title>
+
+    <!-- Recommended meta tags -->
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width,initial-scale=1.0">
+
+    <!-- PyScript CSS -->
+    <link rel="stylesheet" href="https://pyscript.net/releases/2024.1.1/core.css">
+
+    <!-- This script tag bootstraps PyScript -->
+    <script type="module" src="https://pyscript.net/releases/2024.1.1/core.js"></script>
+    <script src="https://www.unpkg.com/ace-builds@latest/src-noconflict/ace.js"></script>
+    <script src="https://www.unpkg.com/ace-builds@latest/src-noconflict/ext-language_tools.js"></script>
+    <script src="https://www.unpkg.com/ace-builds@latest/src-noconflict/theme-chrome.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/ace-linters"></script>
+
+    <script type="module">
+        const loading = document.getElementById('loading');
+        addEventListener('py:ready', () => loading.close());
+        loading.showModal();
+    </script>
+
+    <link rel="icon" href="./image/data_catering_transparent.svg"/>
+    <link rel="stylesheet" href="./src/css/main.css" type="text/css"/>
+    <link rel="stylesheet" href="./src/css/pyscript.css" type="text/css"/>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
+</head>
+
+<body>
+<dialog id="loading">
+    <h1>Loading...</h1>
+</dialog>
+
+<div class="top-banner">
+  <span>
+      <div class="logo-title">
+        <a href="https://data.catering/"><img class="logo" src="./image/data_catering_transparent.svg" alt="logo"/></a>
+        <b>Data Contract Playground</b>
+      </div>
+  </span>
+    <a class="github-button" href="https://github.com/data-catering/data-contract-playground"
+       data-color-scheme="no-preference: light; light: light; dark: dark;" data-icon="octicon-star" data-size="large"
+       data-show-count="true" aria-label="Star data-catering/data-contract-playground on GitHub">Star</a>
+</div>
+<nav class="topnav">
+    <a href="export.html">Export</a>
+    <a href="index.html">Validate</a>
+</nav>
+
+<div>
+    <section>
+        <main>
+            <div class="convert-container">
+                <div class="convert-text">
+                    <p>Powered by <a href="https://github.com/datacontract/datacontract-cli">datacontract-cli</a></p>
+                </div>
+                <div class="convert-button-container">
+                    <button id="convert-btn" type="button" py-click="export_to_output_type">
+                        Convert
+                    </button>
+                </div>
+            </div>
+            <div class="code-containers">
+                <div class="code-container code-container-input" id="convert-input-container">
+                    <div class="code-container-options">
+                        <label for="convert-input-type">Choose input type:</label>
+
+                        <select name="convert-input-type" id="convert-input-type"></select>
+                    </div>
+                    <div class="code" id="input-yaml"></div>
+                </div>
+                <div class="code-container code-container-output" id="convert-output-container">
+                    <div class="code-container-options">
+                        <label for="convert-output-type">Choose output type:</label>
+
+                        <select name="convert-output-type" id="convert-output-type"></select>
+                    </div>
+                    <div class="code" id="output-text"></div>
+                </div>
+            </div>
+        </main>
+        <script type="py" src="main.py" config="./pyscript.toml"></script>
+    </section>
+</div>
+
+<footer>
+    <div class="footer-container">
+        <a class="link footer-left" href="index.html">Data Contract Playground</a>
+        <div class="footer-icons">
+            <ul>
+                <li><a href="https://github.com/data-catering/data-contract-playground" target="_blank"><i
+                        class="fa fa-github fa-2x"></i></a></li>
+                <li><a href="https://www.linkedin.com/in/peter-flook/" target="_blank"><i
+                        class="fa fa-linkedin fa-2x"></i></a></li>
+                <li><a href="https://medium.com/@pflooky" target="_blank"><i class="fa fa-medium fa-2x"></i></a></li>
+            </ul>
+        </div>
+        <a class="link footer-right" href="https://data.catering/about/#contact" target="_blank">contact</a>
+    </div>
+</footer>
+<script type="module" src="src/export.js"></script>
+<script async defer src="https://buttons.github.io/buttons.js"></script>
+</body>
+</html>
diff --git a/index.html b/index.html
@@ -30,6 +30,7 @@
        data-show-count="true" aria-label="Star data-catering/data-contract-playground on GitHub">Star</a>
 </div>
 <nav class="topnav">
+    <a href="export.html">Export</a>
     <a href="index.html">Validate</a>
 </nav>
 <div class="data-contract-type-container">
@@ -62,8 +63,7 @@
     </div>
 </div>
 <div id="yaml-container">
-    <div class="code" id="base-yaml">
-    </div>
+    <div class="code" id="base-yaml"></div>
 </div>
 <footer>
     <div class="footer-container">
@@ -80,7 +80,7 @@
         <a class="link footer-right" href="https://data.catering/about/#contact" target="_blank">contact</a>
     </div>
 </footer>
-<script src="src/index.js"></script>
+<script type="module" src="src/index.js"></script>
 <script async defer src="https://buttons.github.io/buttons.js"></script>
 </body>
 </html>