chore: pin harness ff version range, add warning if dataset name pass…

…ed as user kwarg in factory

pyproject.toml

@@ -9,7 +9,7 @@ dependencies = [
     "sqlmesh",
     "dlt[duckdb]",
     "typer[all]>=0.8.0",
-    "harness-featureflags>=1.2.0",
+    "harness-featureflags>=1.2.0,<1.6.1",
     "python-dotenv>=1.0.0",
     "pex>=2.1.100,<2.2.0",
     "pydantic>=2.5.0,<3.0.0",

src/cdf/__init__.py

@@ -12,7 +12,7 @@
 import cdf.core.context as context
 import cdf.core.logger as logger
 from cdf.core.project import Project, Workspace, load_project
-from cdf.core.runtime.pipeline import pipeline_factory as pipeline
+from cdf.core.runtime import pipeline_factory as pipeline
 from cdf.types import M, PathLike
 
 

src/cdf/core/runtime/__init__.py

@@ -1,5 +1,5 @@
 from cdf.core.runtime.notebook import execute_notebook_specification
-from cdf.core.runtime.pipeline import execute_pipeline_specification
+from cdf.core.runtime.pipeline import execute_pipeline_specification, pipeline_factory
 from cdf.core.runtime.publisher import execute_publisher_specification
 from cdf.core.runtime.script import execute_script_specification
 
@@ -8,4 +8,5 @@
     "execute_pipeline_specification",
     "execute_publisher_specification",
     "execute_script_specification",
+    "pipeline_factory",
 ]

src/cdf/core/runtime/pipeline.py

@@ -273,11 +273,18 @@ def pipeline_factory(**user_options: t.Any) -> RuntimePipeline:
 
     Args:
         **user_options: Kwargs which override the context options. All dlt.pipeline options can
-            be passed here except _impl_cls.
+            be passed here except _impl_cls. Passing dataset_name here will raise a warning.
 
     Raises:
         ValueError if the runtime context is not set.
     """
+    # NOTE: consider Tracer class which tracks dataset_name + pipeline_name combinations in the factory
+    # and stored them in a set? Same in pipeline run/load methods
+    if "dataset_name" in user_options:
+        logger.warning(
+            "Using dataset_name in a cdf pipeline should only be done if you know what you are doing."
+            " cdf will automatically manage the dataset name for you and relies on deterministic naming."
+        )
     runtime = CONTEXT.get()
     context_options = {
         "pipeline_name": runtime.pipeline_name,