🚚 Moved orchiestration folder into src/viadot

src/orchestration/prefect_viadot/flows/cloud_for_customers.py → src/viadot/orchestration/prefect/flows/cloud_for_customers.py

@@ -2,11 +2,12 @@
 
 from typing import Any, Literal
 
-from orchestration.prefect_viadot.tasks import (
+from viadot.orchestration.prefect.tasks import (
     cloud_for_customers_to_df,
     df_to_adls,
     df_to_databricks,
 )
+
 from prefect import flow
 
 

src/orchestration/prefect_viadot/flows/exchange_rates_to_adls.py → src/viadot/orchestration/prefect/flows/exchange_rates_to_adls.py

@@ -3,7 +3,8 @@
 from datetime import datetime
 from typing import Literal
 
-from orchestration.prefect_viadot.tasks import df_to_adls, exchange_rates_to_df
+from viadot.orchestration.prefect.tasks import df_to_adls, exchange_rates_to_df
+
 from prefect import flow
 
 Currency = Literal[

src/orchestration/prefect_viadot/flows/exchange_rates_to_databricks.py → src/viadot/orchestration/prefect/flows/exchange_rates_to_databricks.py

@@ -3,7 +3,8 @@
 from datetime import datetime
 from typing import Literal
 
-from orchestration.prefect_viadot.tasks import df_to_databricks, exchange_rates_to_df
+from viadot.orchestration.prefect.tasks import df_to_databricks, exchange_rates_to_df
+
 from prefect import flow
 
 Currency = Literal[

src/orchestration/prefect_viadot/flows/sap_to_redshift_spectrum.py → src/viadot/orchestration/prefect/flows/sap_to_redshift_spectrum.py

@@ -2,7 +2,8 @@
 
 from typing import Any, Literal
 
-from orchestration.prefect_viadot.tasks import df_to_redshift_spectrum, sap_rfc_to_df
+from viadot.orchestration.prefect.tasks import df_to_redshift_spectrum, sap_rfc_to_df
+
 from prefect import flow
 
 

src/orchestration/prefect_viadot/flows/sharepoint_to_adls.py → src/viadot/orchestration/prefect/flows/sharepoint_to_adls.py

@@ -1,6 +1,7 @@
 """Flows for pulling data from/into Sharepoint."""
 
-from orchestration.prefect_viadot.tasks import df_to_adls, sharepoint_to_df
+from viadot.orchestration.prefect.tasks import df_to_adls, sharepoint_to_df
+
 from prefect import flow
 
 

src/orchestration/prefect_viadot/flows/sharepoint_to_databricks.py → src/viadot/orchestration/prefect/flows/sharepoint_to_databricks.py

@@ -3,7 +3,8 @@
 import contextlib
 from typing import Literal
 
-from orchestration.prefect_viadot.tasks import df_to_databricks, sharepoint_to_df
+from viadot.orchestration.prefect.tasks import df_to_databricks, sharepoint_to_df
+
 from prefect import flow
 
 

src/orchestration/prefect_viadot/flows/sharepoint_to_redshift_spectrum.py → src/viadot/orchestration/prefect/flows/sharepoint_to_redshift_spectrum.py

@@ -2,7 +2,8 @@
 
 from typing import Any, Literal
 
-from orchestration.prefect_viadot.tasks import df_to_redshift_spectrum, sharepoint_to_df
+from viadot.orchestration.prefect.tasks import df_to_redshift_spectrum, sharepoint_to_df
+
 from prefect import flow
 
 

src/orchestration/prefect_viadot/flows/sharepoint_to_s3.py → src/viadot/orchestration/prefect/flows/sharepoint_to_s3.py

@@ -2,10 +2,11 @@
 
 from typing import Any
 
-from orchestration.prefect_viadot.tasks import s3_upload_file, sharepoint_download_file
-from prefect import flow
+from viadot.orchestration.prefect.tasks import s3_upload_file, sharepoint_download_file
 from viadot.sources.sharepoint import SharepointCredentials
 
+from prefect import flow
+
 
 @flow(
     name="extract--sharepoint--s3",

src/orchestration/prefect_viadot/flows/transform.py → src/viadot/orchestration/prefect/flows/transform.py

@@ -3,8 +3,9 @@
 import os
 import shutil
 
-from orchestration.prefect_viadot.tasks import clone_repo, dbt_task
-from orchestration.prefect_viadot.utils import get_credentials
+from viadot.orchestration.prefect.tasks import clone_repo, dbt_task
+from viadot.orchestration.prefect.utils import get_credentials
+
 from prefect import flow, task
 
 

src/orchestration/prefect_viadot/flows/transform_and_catalog.py → src/viadot/orchestration/prefect/flows/transform_and_catalog.py

@@ -4,8 +4,9 @@
 from pathlib import Path
 from typing import Literal
 
-from orchestration.prefect_viadot.tasks import clone_repo, dbt_task, luma_ingest_task
-from orchestration.prefect_viadot.utils import get_credentials
+from viadot.orchestration.prefect.tasks import clone_repo, dbt_task, luma_ingest_task
+from viadot.orchestration.prefect.utils import get_credentials
+
 from prefect import allow_failure, flow, task
 
 

src/orchestration/prefect_viadot/tasks/adls.py → src/viadot/orchestration/prefect/tasks/adls.py

@@ -4,8 +4,9 @@
 from typing import Any
 
 import pandas as pd
-from orchestration.prefect_viadot.exceptions import MissingSourceCredentialsError
-from orchestration.prefect_viadot.utils import get_credentials
+from viadot.orchestration.prefect.exceptions import MissingSourceCredentialsError
+from viadot.orchestration.prefect.utils import get_credentials
+
 from prefect import task
 
 with contextlib.suppress(ImportError):

src/orchestration/prefect_viadot/tasks/cloud_for_customers.py → src/viadot/orchestration/prefect/tasks/cloud_for_customers.py

@@ -3,10 +3,11 @@
 from typing import Any
 
 import pandas as pd
-from orchestration.prefect_viadot.utils import get_credentials
-from prefect import task
+from viadot.orchestration.prefect.utils import get_credentials
 from viadot.sources import CloudForCustomers
 
+from prefect import task
+
 
 @task(retries=3, retry_delay_seconds=10, timeout_seconds=60 * 60)
 def cloud_for_customers_to_df(  # noqa: PLR0913, PLR0917

src/orchestration/prefect_viadot/tasks/databricks.py → src/viadot/orchestration/prefect/tasks/databricks.py

@@ -4,13 +4,14 @@
 from typing import Any, Literal
 
 import pandas as pd
+
 from prefect import task
 
 with contextlib.suppress(ImportError):
     from viadot.sources import Databricks
 
-from orchestration.prefect_viadot.exceptions import MissingSourceCredentialsError
-from orchestration.prefect_viadot.utils import get_credentials
+from viadot.orchestration.prefect.exceptions import MissingSourceCredentialsError
+from viadot.orchestration.prefect.utils import get_credentials
 
 
 @task(retries=3, retry_delay_seconds=10, timeout_seconds=60 * 60)

src/orchestration/prefect_viadot/tasks/dbt.py → src/viadot/orchestration/prefect/tasks/dbt.py

@@ -4,7 +4,8 @@
 import os
 from typing import Any
 
-from orchestration.prefect_viadot.utils import shell_run_command
+from viadot.orchestration.prefect.utils import shell_run_command
+
 from prefect import task
 from prefect.logging import get_run_logger
 

src/orchestration/prefect_viadot/tasks/exchange_rates.py → src/viadot/orchestration/prefect/tasks/exchange_rates.py

@@ -4,11 +4,12 @@
 from typing import Any, Literal
 
 import pandas as pd
-from orchestration.prefect_viadot.exceptions import MissingSourceCredentialsError
-from orchestration.prefect_viadot.utils import get_credentials
-from prefect import task
+from viadot.orchestration.prefect.exceptions import MissingSourceCredentialsError
+from viadot.orchestration.prefect.utils import get_credentials
 from viadot.sources import ExchangeRates
 
+from prefect import task
+
 Currency = Literal[
     "USD", "EUR", "GBP", "CHF", "PLN", "DKK", "COP", "CZK", "SEK", "NOK", "ISK"
 ]

src/orchestration/prefect_viadot/tasks/git.py → src/viadot/orchestration/prefect/tasks/git.py

@@ -5,7 +5,8 @@
 from typing import Any
 
 import pygit2
-from orchestration.prefect_viadot.utils import get_credentials
+from viadot.orchestration.prefect.utils import get_credentials
+
 from prefect import get_run_logger, task
 
 

src/orchestration/prefect_viadot/tasks/luma.py → src/viadot/orchestration/prefect/tasks/luma.py

@@ -5,7 +5,8 @@
 from pathlib import Path
 from typing import Any, Literal
 
-from orchestration.prefect_viadot.utils import shell_run_command
+from viadot.orchestration.prefect.utils import shell_run_command
+
 from prefect import task
 from prefect.logging import get_run_logger
 

src/orchestration/prefect_viadot/tasks/sap_rfc.py → src/viadot/orchestration/prefect/tasks/sap_rfc.py

@@ -4,13 +4,14 @@
 from typing import Any
 
 import pandas as pd
+
 from prefect import task
 from prefect.logging import get_run_logger
 
 with contextlib.suppress(ImportError):
     from viadot.sources import SAPRFC, SAPRFCV2
-from orchestration.prefect_viadot.exceptions import MissingSourceCredentialsError
-from orchestration.prefect_viadot.utils import get_credentials
+from viadot.orchestration.prefect.exceptions import MissingSourceCredentialsError
+from viadot.orchestration.prefect.utils import get_credentials
 
 
 @task(retries=3, retry_delay_seconds=10, timeout_seconds=60 * 60 * 3)

src/orchestration/prefect_viadot/tasks/sharepoint.py → src/viadot/orchestration/prefect/tasks/sharepoint.py

@@ -3,12 +3,13 @@
 from typing import Any
 
 import pandas as pd
-from orchestration.prefect_viadot.exceptions import MissingSourceCredentialsError
-from orchestration.prefect_viadot.utils import get_credentials
-from prefect import get_run_logger, task
+from viadot.orchestration.prefect.exceptions import MissingSourceCredentialsError
+from viadot.orchestration.prefect.utils import get_credentials
 from viadot.sources import Sharepoint
 from viadot.sources.sharepoint import SharepointCredentials
 
+from prefect import get_run_logger, task
+
 
 @task(retries=3, retry_delay_seconds=10, timeout_seconds=60 * 60)
 def sharepoint_to_df(

src/orchestration/prefect_viadot/utils.py → src/viadot/orchestration/prefect/utils.py

@@ -15,7 +15,8 @@
     import awswrangler as wr
 from anyio import open_process
 from anyio.streams.text import TextReceiveStream
-from orchestration.prefect_viadot.exceptions import MissingPrefectBlockError
+from viadot.orchestration.prefect.exceptions import MissingPrefectBlockError
+
 from prefect.client.orchestration import PrefectClient
 from prefect.settings import PREFECT_API_KEY, PREFECT_API_URL
 

tests/orchestration/prefect-viadot_tests/flows/test_cloud_for_customers.py

@@ -1,4 +1,4 @@
-from orchestration.prefect_viadot.flows import (
+from viadot.orchestration.prefect.flows import (
     cloud_for_customers_to_adls,
     cloud_for_customers_to_databricks,
 )

tests/orchestration/prefect-viadot_tests/flows/test_exchange_rates.py

@@ -1,4 +1,4 @@
-from orchestration.prefect_viadot.flows import (
+from viadot.orchestration.prefect.flows import (
     exchange_rates_to_adls,
     exchange_rates_to_databricks,
 )

tests/orchestration/prefect-viadot_tests/flows/test_sap_to_redshift_spectrum.py

@@ -1,6 +1,6 @@
 import os
 
-from orchestration.prefect_viadot.flows import sap_to_redshift_spectrum
+from viadot.orchestration.prefect.flows import sap_to_redshift_spectrum
 from viadot.sources import RedshiftSpectrum
 
 S3_BUCKET = os.environ.get("S3_BUCKET")

tests/orchestration/prefect-viadot_tests/flows/test_sharepoint_to_adls.py

@@ -1,4 +1,4 @@
-from orchestration.prefect_viadot.flows import sharepoint_to_adls
+from viadot.orchestration.prefect.flows import sharepoint_to_adls
 from viadot.sources import AzureDataLake
 
 

tests/orchestration/prefect-viadot_tests/flows/test_sharepoint_to_redshift_spectrum.py

@@ -1,6 +1,6 @@
 import os
 
-from orchestration.prefect_viadot.flows import sharepoint_to_redshift_spectrum
+from viadot.orchestration.prefect.flows import sharepoint_to_redshift_spectrum
 from viadot.sources import RedshiftSpectrum
 
 

tests/orchestration/prefect-viadot_tests/flows/test_sharepoint_to_s3.py

@@ -1,7 +1,7 @@
 import os
 from pathlib import Path
 
-from orchestration.prefect_viadot.flows import sharepoint_to_s3
+from viadot.orchestration.prefect.flows import sharepoint_to_s3
 from viadot.sources import S3
 
 S3_BUCKET = os.environ.get("S3_BUCKET")

tests/orchestration/prefect-viadot_tests/flows/test_transform_and_catalog.py

@@ -1,4 +1,4 @@
-from orchestration.prefect_viadot.flows import transform_and_catalog
+from viadot.orchestration.prefect.flows import transform_and_catalog
 
 
 def test_transform_and_catalog_model(dbt_repo_url, LUMA_URL):

tests/orchestration/prefect-viadot_tests/integration/test_adls.py

@@ -1,7 +1,7 @@
 import pandas as pd
 import pytest
-from orchestration.prefect_viadot.tasks import df_to_adls
 from prefect import flow, task
+from viadot.orchestration.prefect.tasks import df_to_adls
 
 try:
     from viadot.sources import AzureDataLake

tests/orchestration/prefect-viadot_tests/integration/test_databricks.py

@@ -14,7 +14,7 @@
 if not _databricks_installed:
     pytest.skip("Databricks source not installed", allow_module_level=True)
 
-from orchestration.prefect_viadot.tasks import df_to_databricks
+from viadot.orchestration.prefect.tasks import df_to_databricks
 
 TEST_SCHEMA = "test_viadot_schema"
 TEST_TABLE = "test"

tests/orchestration/prefect-viadot_tests/integration/test_dbt.py

@@ -1,5 +1,5 @@
-from orchestration.prefect_viadot.tasks import dbt_task
 from prefect import flow
+from viadot.orchestration.prefect.tasks import dbt_task
 
 
 def test_dbt_task():

tests/orchestration/prefect-viadot_tests/integration/test_exchange_rates_tasks.py

@@ -1,4 +1,4 @@
-from orchestration.prefect_viadot.tasks import exchange_rates_to_df
+from viadot.orchestration.prefect.tasks import exchange_rates_to_df
 
 expected_df = [
     [

tests/orchestration/prefect-viadot_tests/integration/test_git.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 
 from loguru import logger
-from orchestration.prefect_viadot.tasks import clone_repo
+from viadot.orchestration.prefect.tasks import clone_repo
 
 
 def test_clone_repo_private(AZURE_REPO_URL):

tests/orchestration/prefect-viadot_tests/integration/test_luma.py

@@ -1,6 +1,6 @@
 import pytest
 from loguru import logger
-from orchestration.prefect_viadot.tasks import luma_ingest_task
+from viadot.orchestration.prefect.tasks import luma_ingest_task
 
 
 @pytest.mark.asyncio()

tests/orchestration/prefect-viadot_tests/integration/test_redshift_spectrum.py

@@ -2,8 +2,8 @@
 
 import pandas as pd
 import pytest
-from orchestration.prefect_viadot.tasks import df_to_redshift_spectrum
 from prefect import flow
+from viadot.orchestration.prefect.tasks import df_to_redshift_spectrum
 
 try:
     from viadot.sources import RedshiftSpectrum

tests/orchestration/prefect-viadot_tests/integration/test_s3.py

@@ -3,8 +3,8 @@
 
 import pandas as pd
 import pytest
-from orchestration.prefect_viadot.tasks import s3_upload_file
 from prefect import flow
+from viadot.orchestration.prefect.tasks import s3_upload_file
 
 try:
     from viadot.sources import S3

tests/orchestration/prefect-viadot_tests/integration/test_sap_rfc.py

@@ -1,5 +1,5 @@
-from orchestration.prefect_viadot.tasks import sap_rfc_to_df
 from prefect import flow
+from viadot.orchestration.prefect.tasks import sap_rfc_to_df
 
 
 def test_sap_rfc_to_df():

tests/orchestration/prefect-viadot_tests/integration/test_sharepoint_tasks.py

@@ -1,11 +1,11 @@
 import os
 from pathlib import Path
 
-from orchestration.prefect_viadot.tasks import (
+from prefect import flow
+from viadot.orchestration.prefect.tasks import (
     sharepoint_download_file,
     sharepoint_to_df,
 )
-from prefect import flow
 
 
 def test_to_df(sharepoint_url, sharepoint_config_key):