Update to latest version of datacontract-cli, add in sqlalchemy as ne…

…w export type, add in ODCS v3 to validate

README.md

@@ -32,6 +32,7 @@ Current formats supported:
 - SodaCL
 - SQL
 - SQL Query
+- SQLAlchemy
 - Terraform
 
 ## Validate

example/dbml/dbml.txt

@@ -0,0 +1,26 @@
+Project "Orders Latest" {
+    Note: '''Successful customer orders in the webshop. 
+All orders since 2020-01-01. 
+Orders with their line items are in their current state (no history included).
+'''
+}
+
+Table test.orders {
+    "order_id" "text" [pk,unique,not null,Note: "An internal ID that identifies an order in the online shop."]
+"order_timestamp" "timestamp" [not null,Note: "The business timestamp in UTC when the order was successfully registered in the source system and the payment was successful."]
+"order_total" "record" [not null,Note: "Total amount the smallest monetary unit (e.g., cents)."]
+"customer_id" "text" [null,Note: "Unique identifier for the customer."]
+"customer_email_address" "text" [not null,Note: "The email address, as entered by the customer. The email address was not verified."]
+"processed_timestamp" "timestamp" [not null,Note: "The timestamp when the record was processed by the data platform."]
+Note: "One record per order. Includes cancelled and deleted orders."
+}
+
+
+Table orders.line_items {
+"lines_item_id" "text" [pk,unique,not null,Note: "Primary key of the lines_item_id table"]
+"order_id" "text" [null,Note: "An internal ID that identifies an order in the online shop."]
+"sku" "text" [null,Note: "The purchased article number"]
+Note: "A single article that is part of an order."
+}
+
+Ref: orders.line_items.order_id > test.orders.order_id

index.html

@@ -63,7 +63,8 @@
     <select name="data-contract-type" id="data-contract-type">
         <option value="dataContractSpecification">Data Contract Specification</option>
         <option value="dataProductSpecification">Data Product Specification</option>
-        <option value="odcs" selected>Open Data Contract Standard (ODCS)</option>
+        <option value="odcs" selected>Open Data Contract Standard v2 (ODCS)</option>
+        <option value="odcs-v3" selected>Open Data Contract Standard v3 (ODCS)</option>
     </select>
     <div id="github-links" class="github-links-container">
         <div id="odcs-github-link" class="github-link">
@@ -72,6 +73,12 @@
                data-size="large" data-show-count="true" data-hello="world" aria-description="odcs"
                aria-label="Star bitol-io/open-data-contract-standard on GitHub">Star</a>
         </div>
+        <div id="odcs-v3-github-link" class="github-link">
+            <a class="github-button" href="https://github.com/bitol-io/open-data-contract-standard"
+               data-color-scheme="no-preference: light; light: light; dark: dark;" data-icon="octicon-star"
+               data-size="large" data-show-count="true" data-hello="world" aria-description="odcs"
+               aria-label="Star bitol-io/open-data-contract-standard on GitHub">Star</a>
+        </div>
         <div id="dataContractSpecification-github-link" class="github-link">
             <a class="github-button" href="https://github.com/datacontract/datacontract-specification"
                data-color-scheme="no-preference: light; light: light; dark: dark;" data-icon="octicon-star"

pyscript.toml

@@ -19,6 +19,7 @@ packages = [
 "example/avro/orders.avsc" = "./example/avro/orders.avsc"
 "example/bigquery/complete_table_schema.json" = "./example/bigquery/complete_table_schema.json"
 "example/dbt/manifest_jaffle_duckdb.json" = "./example/dbt/manifest_jaffle_duckdb.json"
+"example/dbml/dbml.txt" = "./example/dbml/dbml.txt"
 "example/jsonschema/orders_union-types.json" = "./example/jsonschema/orders_union-types.json"
 "example/sql/customer.sql" = "./example/sql/customer.sql"
 "example/unity/unity_table_schema.json" = "./example/unity/unity_table_schema.json"
@@ -46,11 +47,13 @@ files = [
   "export/spark_converter.py",
   "export/sql_converter.py",
   "export/sql_type_converter.py",
+  "export/sqlalchemy_converter.py",
   "export/terraform_converter.py",
   "imports/importer.py",
   "imports/importer_factory.py",
   "imports/avro_importer.py",
   "imports/bigquery_importer.py",
+  "imports/dbml_importer.py",
   "imports/dbt_importer.py",
   "imports/jsonschema_importer.py",
   "imports/odcs_importer.py",

src/js/config.js

@@ -1,12 +1,19 @@
 
 export const defaultJsonSchemaName = "odcs"
+export const defaultValidateJsonSchemaName = "odcs-v3"
 
 export const odcsDetails = {
     schemaUrl: "https://raw.githubusercontent.com/bitol-io/open-data-contract-standard/main/schema/odcs-json-schema-latest.json",
     exampleUrl: "https://raw.githubusercontent.com/bitol-io/open-data-contract-standard/main/docs/examples/all/full-example.odcs.yaml",
     githubUrl: "https://github.com/bitol-io/open-data-contract-standard"
 }
 
+export const odcsV3Details = {
+    schemaUrl: "https://raw.githubusercontent.com/bitol-io/open-data-contract-standard/dev_v3/schema/odcs-json-schema-v3.0.0.json",
+    exampleUrl: "https://raw.githubusercontent.com/bitol-io/open-data-contract-standard/dev_v3/docs/examples/all/full-example.odcs.yaml",
+    githubUrl: "https://github.com/bitol-io/open-data-contract-standard"
+}
+
 export const dataContractSpecificationDetails = {
     schemaUrl: "https://raw.githubusercontent.com/datacontract/datacontract-specification/main/datacontract.schema.json",
     exampleUrl: "https://raw.githubusercontent.com/datacontract/datacontract-specification/main/examples/covid-cases/datacontract.yaml",

src/js/index.js

@@ -2,12 +2,14 @@ import {setJsonSchema, setupEditorSession} from "./util.js";
 import {
     dataContractSpecificationDetails,
     dataProductSpecificationDetails,
-    defaultJsonSchemaName,
-    odcsDetails
+    defaultValidateJsonSchemaName,
+    odcsDetails,
+    odcsV3Details
 } from "./config.js";
 
 const jsonSchemaMap = new Map()
 jsonSchemaMap.set("odcs", odcsDetails)
+jsonSchemaMap.set("odcs-v3", odcsV3Details)
 jsonSchemaMap.set("dataContractSpecification", dataContractSpecificationDetails)
 jsonSchemaMap.set("dataProductSpecification", dataProductSpecificationDetails)
 
@@ -20,10 +22,10 @@ async function initAceEditor() {
     ace.require("ace/ext/language_tools")
     setupEditorSession(editor, "ace/mode/yaml")
 
-    githubLinks.dataset.prevValue = defaultJsonSchemaName
-    const githubLink = document.getElementById(`${defaultJsonSchemaName}-github-link`)
+    githubLinks.dataset.prevValue = defaultValidateJsonSchemaName
+    const githubLink = document.getElementById(`${defaultValidateJsonSchemaName}-github-link`)
     githubLink.style.display = "block"
-    await setJsonSchema(jsonSchemaMap, exampleMap, editor, provider, defaultJsonSchemaName)
+    await setJsonSchema(jsonSchemaMap, exampleMap, editor, provider, defaultValidateJsonSchemaName)
 }
 
 function initSelectSchemaListener() {

src/python/main.py

@@ -11,7 +11,7 @@
 from src.python.model import OdcsDetails, DataContractSpecDetails, AvroDetails, AvroIdlDetails, BigQueryDetails, \
     DbmlDetails, DbtDetails, DbtSourcesDetails, DbtStagingSqlDetails, GoDetails, GreatExpectationsDetails, HtmlDetails, \
     JsonSchemaDetails, ProtobufDetails, PydanticModelDetails, RdfDetails, SodaClDetails, SparkDetails, SqlDetails, \
-    SqlQueryDetails, TerraformDetails, DataContractSpec
+    SqlQueryDetails, SqlAlchemyDetails, TerraformDetails, DataContractSpec
 
 convert_output_type = pydom["#convert-output-type"][0]
 ace = window.ace
@@ -23,7 +23,7 @@
     AvroDetails(), AvroIdlDetails(), BigQueryDetails(), DbmlDetails(), DbtDetails(), DbtSourcesDetails(),
     DbtStagingSqlDetails(), GoDetails(), GreatExpectationsDetails(), HtmlDetails(), JsonSchemaDetails(),
     OdcsDetails(), ProtobufDetails(), PydanticModelDetails(), RdfDetails(), SodaClDetails(), SparkDetails(),
-    SqlDetails(), SqlQueryDetails(), TerraformDetails()
+    SqlDetails(), SqlQueryDetails(), SqlAlchemyDetails(), TerraformDetails()
 ]
 
 data_contract_examples = {}

src/python/model.py

@@ -86,6 +86,7 @@ class DataContractSpecDetails(DataSourceDetails):
 class DbmlDetails(DataSourceDetails):
     name = "dbml"
     display_name = "DBML"
+    example_file = "example/dbml/dbml.txt"
 
 
 class DbtDetails(DataSourceDetails):
@@ -181,6 +182,12 @@ class SqlQueryDetails(DataSourceDetails):
     additional_opts = {"editor_mode": "sql"}
 
 
+class SqlAlchemyDetails(DataSourceDetails):
+    name = "sqlalchemy"
+    display_name = "SQLAlchemy"
+    additional_opts = {"editor_mode": "sql"}
+
+
 class TerraformDetails(DataSourceDetails):
     name = "terraform"
     display_name = "Terraform"