Merge pull request #410 from rtdip/develop

v0.6.2

docs/sdk/code-reference/pipelines/destinations/python/delta.md

@@ -0,0 +1,2 @@
+# Write to Delta
+::: src.sdk.python.rtdip_sdk.pipelines.destinations.python.delta

docs/sdk/code-reference/pipelines/sources/python/delta.md

@@ -0,0 +1,2 @@
+# Read from Delta
+::: src.sdk.python.rtdip_sdk.pipelines.sources.python.delta

docs/sdk/code-reference/pipelines/sources/python/delta_sharing.md

@@ -0,0 +1,2 @@
+# Read from Delta with Delta Sharing
+::: src.sdk.python.rtdip_sdk.pipelines.sources.python.delta_sharing

docs/sdk/code-reference/pipelines/transformers/spark/raw_forecast_to_weather_data_model.md

@@ -0,0 +1,2 @@
+# Convert Forecast Raw JSON data to the Weather Data Model
+::: src.sdk.python.rtdip_sdk.pipelines.transformers.spark.raw_forecast_to_weather_data_model

docs/sdk/pipelines/components.md

@@ -45,7 +45,6 @@ Sources are components that connect to source systems and extract data from them
 |[PJM Historical Load ISO](../code-reference/pipelines/sources/spark/iso/pjm_historical_load_iso.md) ||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
 |[Weather Forecast API V1](../code-reference/pipelines/sources/spark/weather/weather_forecast_api_v1.md)||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
 |[Weather Forecast API V1 Multi](../code-reference/pipelines/sources/spark/weather/weather_forecast_api_v1_multi.md)||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
-
 !!! note "Note"
     This list will dynamically change as the framework is further developed and new components are added.
 
@@ -65,6 +64,7 @@ Transformers are components that perform transformations on data. These will tar
 |[Pandas to PySpark DataFrame Conversion](../code-reference/pipelines/transformers/spark/pandas_to_pyspark.md)||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
 |[PySpark to Pandas DataFrame Conversion](../code-reference/pipelines/transformers/spark/pyspark_to_pandas.md)||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
 |[MISO To Meters Data Model](../code-reference/pipelines/transformers/spark/iso/miso_to_mdm.md)||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
+|[Raw Forecast to Weather Data Model](../code-reference/pipelines/transformers/spark/raw_forecast_to_weather_data_model.md)||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
 |[PJM To Meters Data Model](../code-reference/pipelines/transformers/spark/iso/pjm_to_mdm.md)||:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|:heavy_check_mark:|
 
 !!! note "Note"

environment.yml

@@ -37,7 +37,7 @@ dependencies:
     - azure-keyvault-secrets==4.7.0
     - boto3==1.28.2
     - pyodbc==4.0.39
-    - fastapi==0.100.0
+    - fastapi==0.100.1
     - httpx==0.24.1
     - trio==0.22.1
     - pyspark>=3.3.0,<3.5.0
@@ -53,19 +53,23 @@ dependencies:
     - mkdocs-macros-plugin==1.0.1
     - pygments==2.15.1
     - pymdown-extensions==10.0.1
-    - databricks-sql-connector==2.7.0
+    - databricks-sql-connector==2.8.0
     - databricks-sdk==0.2.1
     - semver==3.0.0
     - xlrd==2.0.1
     - pygithub==1.59.0
     - strawberry-graphql[fastapi,pydantic]==0.194.4
     - web3==6.5.0
+    - twine==4.0.2
     - pip:
         - dependency-injector==4.41.0
         - azure-functions==1.15.0
         - nest_asyncio==1.5.6
         - hvac==1.1.1
-        - langchain==0.0.230
+        - langchain==0.0.247
         - deltalake==0.10.0
         - moto[s3]==4.1.13
+        - build==0.10.0
+        - polars==0.18.8
+        - delta-sharing==0.7.3
 

mkdocs.yml

@@ -160,6 +160,9 @@ nav:
                           - Base Weather: sdk/code-reference/pipelines/sources/spark/weather/base_weather.md
                           - Weather Forecast API V1: sdk/code-reference/pipelines/sources/spark/weather/weather_forecast_api_v1.md
                           - Weather Forecast API V1 Multi: sdk/code-reference/pipelines/sources/spark/weather/weather_forecast_api_v1_multi.md
+                  - Python:
+                      - Delta: sdk/code-reference/pipelines/sources/python/delta.md
+                      - Delta Sharing: sdk/code-reference/pipelines/sources/python/delta_sharing.md
               - Transformers:
                   - Spark:
                       - Binary To String: sdk/code-reference/pipelines/transformers/spark/binary_to_string.md
@@ -174,6 +177,9 @@ nav:
                       - ISO:
                           - MISO To Meters Data Model: sdk/code-reference/pipelines/transformers/spark/iso/miso_to_mdm.md
                           - PJM To Meters Data Model: sdk/code-reference/pipelines/transformers/spark/iso/pjm_to_mdm.md
+                      - Weather:
+                          - Raw Forecast To Weather Data Model: sdk/code-reference/pipelines/transformers/spark/raw_forecast_to_weather_data_model.md
+
               - Destinations:
                   - Spark:
                       - Delta: sdk/code-reference/pipelines/destinations/spark/delta.md
@@ -183,6 +189,8 @@ nav:
                       - Kinesis: sdk/code-reference/pipelines/destinations/spark/kinesis.md
                       - Rest API: sdk/code-reference/pipelines/destinations/spark/rest_api.md
                       - Process Control Data Model To Delta: sdk/code-reference/pipelines/destinations/spark/pcdm_to_delta.md
+                  - Python:
+                      - Delta: sdk/code-reference/pipelines/destinations/python/delta.md
                   - Blockchain:
                       - EVM: sdk/code-reference/pipelines/destinations/blockchain/evm.md
 

setup.py

@@ -29,7 +29,7 @@
 long_description = (here / "PYPI-README.md").read_text()
 
 INSTALL_REQUIRES = [
-  "databricks-sql-connector==2.7.0",
+  "databricks-sql-connector==2.8.0",
   "azure-identity==1.12.0",
   "pyodbc==4.0.39",
   "pandas==1.5.2",
@@ -40,7 +40,7 @@
   "grpcio>=1.48.1",
   "grpcio-status>=1.48.1",
   "googleapis-common-protos>=1.56.4",
-  "langchain==0.0.230",
+  "langchain==0.0.247",
   "openai==0.27.8"
 ]
 

src/api/FastAPIApp/__init__.py

@@ -67,6 +67,7 @@
     openapi_url="/api/openapi.json",
     docs_url=None,
     redoc_url=None,
+    license_info={"name": "Apache License 2.0", "identifier": "Apache-2.0"}
 )
 
 app.add_middleware(GZipMiddleware, minimum_size=1000)

src/api/README.md

@@ -35,6 +35,13 @@ Ensure that you setup the **local.settings.json** file with the relevant paramet
 |DATABRICKS_SQL_SERVER_HOSTNAME|adb-xxxxx.x.azuredatabricks.net|
 |DATABRICKS_SQL_HTTP_PATH|/sql/1.0/warehouses/xxx|
 
+Please also ensure to install all the turbodbc requirements for your machine by reviewing the [installation instructions](https://turbodbc.readthedocs.io/en/latest/pages/getting_started.html) of turbodbc. On a macbook, this includes executing the following commands:
+
+```bash
+brew install llvm
+brew install boost
+```
+
 ### Swagger and Redoc
 
 Fast API provides endpoints for Swagger and Redoc pages. Ensure that you review these pages after any updates to confirm they are working as expected.

src/api/requirements.txt

@@ -1,12 +1,12 @@
 # Do not include azure-functions-worker as it may conflict with the Azure Functions platform
 azure-functions==1.15.0
-fastapi==0.100.0
+fastapi==0.100.1
 nest_asyncio==1.5.6
 strawberry-graphql[fastapi,pydantic]==0.194.4
 turbodbc==4.5.10
 pyodbc==4.0.39
 importlib_metadata>=1.0.0
-databricks-sql-connector==2.7.0
+databricks-sql-connector==2.8.0
 azure-identity==1.12.0
 oauthlib>=3.2.2
 pandas==1.5.2

src/api/v1/graphql.py

@@ -40,8 +40,8 @@ async def raw_get(
       data_type: str = Query(..., description="Data Type", examples={"float": {"value": "float"}, "integer": {"value": "integer"}, "string": {"value": "string"}}), 
       tag_name: List[str] = Query(..., description="Tag Name"),
       include_bad_data: bool = Query(True, description="Include or remove Bad data points"),             
-      start_date: date = Query(..., description="Start Date", example="2022-01-01"),
-      end_date: date = Query(..., description="End Date", example="2022-01-02"),
+      start_date: date = Query(..., description="Start Date", examples="2022-01-01"),
+      end_date: date = Query(..., description="End Date", examples="2022-01-02"),
       authorization: str = Header(None, include_in_schema=False),
       # authorization: str = Depends(oauth2_scheme)
       ) -> RawResponseQL:

src/api/v1/interpolate.py

@@ -43,15 +43,16 @@ def interpolate_events_get(base_query_parameters, raw_query_parameters, tag_quer
 get_description = """
 ## Interpolate 
 
-Interpolation of raw timeseries data. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/query/interpolate/) for further information.
+Interpolation of raw timeseries data.
 """
 
 @api_v1_router.get(
     path="/events/interpolate",
     name="Interpolate GET",
     description=get_description, 
     tags=["Events"], 
-    responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}}
+    responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}},
+    openapi_extra={"externalDocs": {"description": "RTDIP Interpolation Query Documentation", "url": "https://www.rtdip.io/sdk/code-reference/query/interpolate/"}}
 )
 async def interpolate_get(
         base_query_parameters: BaseQueryParams = Depends(), 
@@ -65,15 +66,16 @@ async def interpolate_get(
 post_description = """
 ## Interpolate 
 
-Interpolation of raw timeseries data via a POST method to enable providing a list of tag names that can exceed url length restrictions via GET Query Parameters. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/query/interpolate/) for further information.
+Interpolation of raw timeseries data via a POST method to enable providing a list of tag names that can exceed url length restrictions via GET Query Parameters.
 """
 
 @api_v1_router.post(
     path="/events/interpolate",
     name="Interpolate POST",
     description=get_description, 
     tags=["Events"], 
-    responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}}
+    responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}},
+    openapi_extra={"externalDocs": {"description": "RTDIP Interpolation Query Documentation", "url": "https://www.rtdip.io/sdk/code-reference/query/interpolate/"}}
 )
 async def interpolate_post(
         base_query_parameters: BaseQueryParams = Depends(), 

src/api/v1/interpolation_at_time.py

@@ -41,15 +41,16 @@ def interpolation_at_time_events_get(base_query_parameters, tag_query_parameters
 get_description = """
 ## Interpolation at Time
 
-Interpolation at Time of raw timeseries data. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/query/interpolation_at_time/) for further information.
+Interpolation at Time of raw timeseries data.
 """
 
 @api_v1_router.get(
     path="/events/interpolationattime",
     name="Interpolation at Time GET",
     description=get_description, 
     tags=["Events"], 
-    responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}}
+    responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}},
+    openapi_extra={"externalDocs": {"description": "RTDIP Interpolation At Time Query Documentation", "url": "https://www.rtdip.io/sdk/code-reference/query/interpolate_at_time/"}}
 )
 async def interpolate_get(
         base_query_parameters: BaseQueryParams = Depends(), 
@@ -61,15 +62,16 @@ async def interpolate_get(
 post_description = """
 ## Interpolation at Time
 
-Interpolation at time of raw timeseries data via a POST method to enable providing a list of tag names that can exceed url length restrictions via GET Query Parameters. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/query/interpolation_at_time/) for further information.
+Interpolation at time of raw timeseries data via a POST method to enable providing a list of tag names that can exceed url length restrictions via GET Query Parameters.
 """
 
 @api_v1_router.post(
     path="/events/interpolationattime",
     name="Interpolation at Time POST",
     description=get_description, 
     tags=["Events"], 
-    responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}}
+    responses={200: {"model": ResampleInterpolateResponse}, 400: {"model": HTTPError}},
+    openapi_extra={"externalDocs": {"description": "RTDIP Interpolation At Time Query Documentation", "url": "https://www.rtdip.io/sdk/code-reference/query/interpolate_at_time/"}}
 )
 async def interpolate_post(
         base_query_parameters: BaseQueryParams = Depends(), 

src/api/v1/metadata.py

@@ -39,7 +39,7 @@ def metadata_retrieval_get(query_parameters, metadata_query_parameters):
 get_description = """
 ## Metadata 
 
-Retrieval of metadata, including UoM, Description and any other possible fields, if available. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/query/metadata/) for further information.
+Retrieval of metadata, including UoM, Description and any other possible fields, if available.
 """
 
 @api_v1_router.get(
@@ -48,15 +48,16 @@ def metadata_retrieval_get(query_parameters, metadata_query_parameters):
     description=get_description,
     tags=["Metadata"], 
     dependencies=[Depends(oauth2_scheme)],
-    responses={200: {"model": MetadataResponse}, 400: {"model": HTTPError}}
+    responses={200: {"model": MetadataResponse}, 400: {"model": HTTPError}},
+    openapi_extra={"externalDocs": {"description": "RTDIP Metadata Query Documentation", "url": "https://www.rtdip.io/sdk/code-reference/query/metadata/"}}
 )
 async def metadata_get(query_parameters: BaseQueryParams = Depends(), metadata_query_parameters: MetadataQueryParams = Depends()):
     return metadata_retrieval_get(query_parameters, metadata_query_parameters)
 
 post_description = """
 ## Metadata 
 
-Retrieval of metadata, including UoM, Description and any other possible fields, if available via a POST method to enable providing a list of tag names that can exceed url length restrictions via GET Query Parameters. Refer to the following [documentation](https://www.rtdip.io/sdk/code-reference/query/metadata/) for further information.
+Retrieval of metadata, including UoM, Description and any other possible fields, if available via a POST method to enable providing a list of tag names that can exceed url length restrictions via GET Query Parameters.
 """
 
 @api_v1_router.post(
@@ -65,7 +66,8 @@ async def metadata_get(query_parameters: BaseQueryParams = Depends(), metadata_q
     description=post_description,
     tags=["Metadata"], 
     dependencies=[Depends(oauth2_scheme)],
-    responses={200: {"model": MetadataResponse}, 400: {"model": HTTPError}}
+    responses={200: {"model": MetadataResponse}, 400: {"model": HTTPError}},
+    openapi_extra={"externalDocs": {"description": "RTDIP Metadata Query Documentation", "url": "https://www.rtdip.io/sdk/code-reference/query/metadata/"}}
 )
 async def metadata_post(query_parameters: BaseQueryParams = Depends(), metadata_query_parameters: TagsBodyParams = Body(default=...)):
     return metadata_retrieval_get(query_parameters, metadata_query_parameters)