diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 169e0e58b7..51de3247fe 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -57,7 +57,7 @@ jobs: echo "No tagged version found, exiting" exit 1 fi - LINK="https://pypi.org/project/flytekit/${VERSION}" + LINK="https://pypi.org/project/flytekitplugins-pod/${VERSION}" for i in {1..60}; do if curl -L -I -s -f ${LINK} >/dev/null; then echo "Found pypi" diff --git a/flytekit/clients/auth/authenticator.py b/flytekit/clients/auth/authenticator.py index f67234f370..b2b82831c7 100644 --- a/flytekit/clients/auth/authenticator.py +++ b/flytekit/clients/auth/authenticator.py @@ -4,6 +4,8 @@ from abc import abstractmethod from dataclasses import dataclass +import click + from . import token_client from .auth_client import AuthorizationClient from .exceptions import AccessTokenNotFoundError, AuthenticationError @@ -255,11 +257,8 @@ def refresh_credentials(self): resp = token_client.get_device_code( self._device_auth_endpoint, self._client_id, self._audience, self._scope, self._http_proxy_url, self._verify ) - print( - f""" -To Authenticate navigate in a browser to the following URL: {resp.verification_uri} and enter code: {resp.user_code} - """ - ) + text = f"To Authenticate, navigate in a browser to the following URL: {click.style(resp.verification_uri, fg='blue', underline=True)} and enter code: {click.style(resp.user_code, fg='blue')}" + click.secho(text) try: # Currently the refresh token is not retreived. We may want to add support for refreshTokens so that # access tokens can be refreshed for once authenticated machines diff --git a/flytekit/clients/friendly.py b/flytekit/clients/friendly.py index 2b15dfbd50..2bae266a53 100644 --- a/flytekit/clients/friendly.py +++ b/flytekit/clients/friendly.py @@ -1018,3 +1018,9 @@ def get_download_signed_url( expires_in=expires_in_pb, ) ) + + def get_data(self, flyte_uri: str) -> _data_proxy_pb2.GetDataResponse: + req = _data_proxy_pb2.GetDataRequest(flyte_url=flyte_uri) + + resp = self._dataproxy_stub.GetData(req, metadata=self._metadata) + return resp diff --git a/flytekit/clients/raw.py b/flytekit/clients/raw.py index e71485b17c..836d5ffa3b 100644 --- a/flytekit/clients/raw.py +++ b/flytekit/clients/raw.py @@ -376,6 +376,15 @@ def get_execution_data(self, get_execution_data_request): """ return self._stub.GetExecutionData(get_execution_data_request, metadata=self._metadata) + def get_execution_metrics(self, get_execution_metrics_request): + """ + Returns metrics partitioning and categorizing the workflow execution time-series. + + :param flyteidl.admin.execution_pb2.WorkflowExecutionGetMetricsRequest get_execution_metrics_request: + :rtype: flyteidl.admin.execution_pb2.WorkflowExecutionGetMetricsResponse + """ + return self._stub.GetExecutionMetrics(get_execution_metrics_request, metadata=self._metadata) + def list_executions_paginated(self, resource_list_request): """ Lists the executions for a given identifier. @@ -570,9 +579,9 @@ def create_upload_location( def create_download_location( self, create_download_location_request: _dataproxy_pb2.CreateDownloadLocationRequest ) -> _dataproxy_pb2.CreateDownloadLocationResponse: - """ - Get a signed url to be used during fast registration - :param flyteidl.service.dataproxy_pb2.CreateDownloadLocationRequest create_download_location_request: - :rtype: flyteidl.service.dataproxy_pb2.CreateDownloadLocationResponse - """ return self._dataproxy_stub.CreateDownloadLocation(create_download_location_request, metadata=self._metadata) + + def create_download_link( + self, create_download_link_request: _dataproxy_pb2.CreateDownloadLinkRequest + ) -> _dataproxy_pb2.CreateDownloadLinkResponse: + return self._dataproxy_stub.CreateDownloadLink(create_download_link_request, metadata=self._metadata) diff --git a/flytekit/clis/sdk_in_container/metrics.py b/flytekit/clis/sdk_in_container/metrics.py new file mode 100644 index 0000000000..40e40a6f70 --- /dev/null +++ b/flytekit/clis/sdk_in_container/metrics.py @@ -0,0 +1,218 @@ +from datetime import datetime + +import rich_click as click +import yaml +from flyteidl.admin.execution_pb2 import WorkflowExecutionGetMetricsRequest +from flyteidl.core.identifier_pb2 import WorkflowExecutionIdentifier + +from flytekit.clis.sdk_in_container.constants import CTX_DOMAIN, CTX_PROJECT +from flytekit.clis.sdk_in_container.helpers import get_and_save_remote_with_click_context + +CTX_DEPTH = "depth" + +_dump_help = """ +The dump command aggregates workflow execution metrics and displays them. This aggregation is meant to provide an easy +to understand breakdown of where time is spent in a hierarchical manner. + +- execution_id refers to the id of the workflow execution +""" + +_explain_help = """ +The explain command prints each individual execution span and the associated timestamps and Flyte entity reference. +This breakdown provides precise information into exactly how and when Flyte processes a workflow execution. + +- execution_id refers to the id of the workflow execution +""" + + +@click.group("metrics") +@click.option( + "-d", + "--depth", + required=False, + type=int, + default=-1, + help="The depth of Flyte entity hierarchy to traverse when computing metrics for this execution", +) +@click.option( + "-p", + "--project", + required=False, + type=str, + default="flytesnacks", + help="The project of the workflow execution", +) +@click.option( + "-d", + "--domain", + required=False, + type=str, + default="development", + help="The domain of the workflow execution", +) +@click.pass_context +def metrics(ctx: click.Context, depth, domain, project): + ctx.obj[CTX_DEPTH] = depth + ctx.obj[CTX_DOMAIN] = domain + ctx.obj[CTX_PROJECT] = project + + +@click.command("dump", help=_dump_help) +@click.argument("execution_id", type=str) +@click.pass_context +def metrics_dump( + ctx: click.Context, + execution_id: str, +): + depth = ctx.obj[CTX_DEPTH] + domain = ctx.obj[CTX_DOMAIN] + project = ctx.obj[CTX_PROJECT] + + # retrieve remote + remote = get_and_save_remote_with_click_context(ctx, project, domain) + sync_client = remote.client + + # retrieve workflow execution metrics + workflow_execution_id = WorkflowExecutionIdentifier(project=project, domain=domain, name=execution_id) + + request = WorkflowExecutionGetMetricsRequest(id=workflow_execution_id, depth=depth) + response = sync_client.get_execution_metrics(request) + + # aggregate spans and print + id, info = aggregate_reference_span(response.span) + yaml.emitter.Emitter.process_tag = lambda self, *args, **kw: None + print(yaml.dump({id: info}, indent=2)) + + +def aggregate_reference_span(span): + id = "" + id_type = span.WhichOneof("id") + if id_type == "workflow_id": + id = span.workflow_id.name + elif id_type == "node_id": + id = span.node_id.node_id + elif id_type == "task_id": + id = span.task_id.retry_attempt + + spans = aggregate_spans(span.spans) + return id, spans + + +def aggregate_spans(spans): + breakdown = {} + + tasks = {} + nodes = {} + workflows = {} + + for span in spans: + id_type = span.WhichOneof("id") + if id_type == "operation_id": + operation_id = span.operation_id + + start_time = datetime.fromtimestamp(span.start_time.seconds + span.start_time.nanos / 1e9) + end_time = datetime.fromtimestamp(span.end_time.seconds + span.end_time.nanos / 1e9) + total_time = (end_time - start_time).total_seconds() + + if operation_id in breakdown: + breakdown[operation_id] += total_time + else: + breakdown[operation_id] = total_time + else: + id, underlying_span = aggregate_reference_span(span) + + if id_type == "workflow_id": + workflows[id] = underlying_span + elif id_type == "node_id": + nodes[id] = underlying_span + elif id_type == "task_id": + tasks[id] = underlying_span + + for operation_id, total_time in underlying_span["breakdown"].items(): + if operation_id in breakdown: + breakdown[operation_id] += total_time + else: + breakdown[operation_id] = total_time + + span = {"breakdown": breakdown} + + if len(tasks) > 0: + span["task_attempts"] = tasks + if len(nodes) > 0: + span["nodes"] = nodes + if len(workflows) > 0: + span["workflows"] = workflows + + return span + + +@click.command("explain", help=_explain_help) +@click.argument("execution_id", type=str) +@click.pass_context +def metrics_explain( + ctx: click.Context, + execution_id: str, +): + depth = ctx.obj[CTX_DEPTH] + domain = ctx.obj[CTX_DOMAIN] + project = ctx.obj[CTX_PROJECT] + + # retrieve remote + remote = get_and_save_remote_with_click_context(ctx, project, domain) + sync_client = remote.client + + # retrieve workflow execution metrics + workflow_execution_id = WorkflowExecutionIdentifier(project=project, domain=domain, name=execution_id) + + request = WorkflowExecutionGetMetricsRequest(id=workflow_execution_id, depth=depth) + response = sync_client.get_execution_metrics(request) + + # print execution spans + print( + "{:25s}{:25s}{:25s} {:>8s} {:s}".format( + "operation", "start_timestamp", "end_timestamp", "duration", "entity" + ) + ) + print("-" * 140) + + print_span(response.span, -1, "") + + +def print_span(span, indent, identifier): + start_time = datetime.fromtimestamp(span.start_time.seconds + span.start_time.nanos / 1e9) + end_time = datetime.fromtimestamp(span.end_time.seconds + span.end_time.nanos / 1e9) + + id_type = span.WhichOneof("id") + span_identifier = "" + + if id_type == "operation_id": + indent_str = "" + for i in range(indent): + indent_str += " " + + print( + "{:25s}{:25s}{:25s} {:7.2f}s {:s}{:s}".format( + span.operation_id, + start_time.strftime("%m-%d %H:%M:%S.%f"), + end_time.strftime("%m-%d %H:%M:%S.%f"), + (end_time - start_time).total_seconds(), + indent_str, + identifier, + ) + ) + + span_identifier = identifier + "/" + span.operation_id + else: + if id_type == "workflow_id": + span_identifier = "workflow/" + span.workflow_id.name + elif id_type == "node_id": + span_identifier = "node/" + span.node_id.node_id + elif id_type == "task_id": + span_identifier = "task/" + str(span.task_id.retry_attempt) + + for under_span in span.spans: + print_span(under_span, indent + 1, span_identifier) + + +metrics.add_command(metrics_dump) +metrics.add_command(metrics_explain) diff --git a/flytekit/clis/sdk_in_container/pyflyte.py b/flytekit/clis/sdk_in_container/pyflyte.py index c2cc4d7681..67bc99ab19 100644 --- a/flytekit/clis/sdk_in_container/pyflyte.py +++ b/flytekit/clis/sdk_in_container/pyflyte.py @@ -12,6 +12,7 @@ from flytekit.clis.sdk_in_container.init import init from flytekit.clis.sdk_in_container.launchplan import launchplan from flytekit.clis.sdk_in_container.local_cache import local_cache +from flytekit.clis.sdk_in_container.metrics import metrics from flytekit.clis.sdk_in_container.package import package from flytekit.clis.sdk_in_container.register import register from flytekit.clis.sdk_in_container.run import run @@ -144,6 +145,7 @@ def main(ctx, pkgs: typing.List[str], config: str, verbose: bool): main.add_command(backfill) main.add_command(serve) main.add_command(build) +main.add_command(metrics) main.add_command(launchplan) main.epilog diff --git a/flytekit/clis/sdk_in_container/run.py b/flytekit/clis/sdk_in_container/run.py index f3d8c661e2..3753b237f9 100644 --- a/flytekit/clis/sdk_in_container/run.py +++ b/flytekit/clis/sdk_in_container/run.py @@ -9,6 +9,7 @@ from dataclasses import dataclass from typing import cast +import cloudpickle import rich_click as click import yaml from dataclasses_json import DataClassJsonMixin @@ -34,7 +35,7 @@ from flytekit.configuration.default_images import DefaultImages from flytekit.core import context_manager from flytekit.core.base_task import PythonTask -from flytekit.core.context_manager import FlyteContext +from flytekit.core.context_manager import FlyteContext, FlyteContextManager from flytekit.core.data_persistence import FileAccessProvider from flytekit.core.type_engine import TypeEngine from flytekit.core.workflow import PythonFunctionWorkflow, WorkflowBase @@ -46,6 +47,7 @@ from flytekit.tools import module_loader, script_mode from flytekit.tools.script_mode import _find_project_root from flytekit.tools.translator import Options +from flytekit.types.pickle.pickle import FlytePickleTransformer REMOTE_FLAG_KEY = "remote" RUN_LEVEL_PARAMS_KEY = "run_level_params" @@ -104,6 +106,19 @@ def convert( raise click.BadParameter(f"parameter should be a valid file path, {value}") +class PickleParamType(click.ParamType): + name = "pickle" + + def convert( + self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context] + ) -> typing.Any: + + uri = FlyteContextManager.current_context().file_access.get_random_local_path() + with open(uri, "w+b") as outfile: + cloudpickle.dump(value, outfile) + return FileParam(filepath=str(pathlib.Path(uri).resolve())) + + class DateTimeType(click.DateTime): _NOW_FMT = "now" @@ -228,7 +243,10 @@ def __init__( if self._literal_type.blob: if self._literal_type.blob.dimensionality == BlobType.BlobDimensionality.SINGLE: - self._click_type = FileParamType() + if self._literal_type.blob.format == FlytePickleTransformer.PYTHON_PICKLE_FORMAT: + self._click_type = PickleParamType() + else: + self._click_type = FileParamType() else: self._click_type = DirParamType() @@ -551,12 +569,25 @@ def get_workflow_command_base_params() -> typing.List[click.Option]: default=False, help="Whether to dump a code snippet instructing how to load the workflow execution using flyteremote", ), + click.Option( + param_decls=["--overwrite-cache", "overwrite_cache"], + required=False, + is_flag=True, + default=False, + help="Whether to overwrite the cache if it already exists", + ), + click.Option( + param_decls=["--envs", "envs"], + required=False, + type=JsonParamType(), + help="Environment variables to set in the container", + ), ] def load_naive_entity(module_name: str, entity_name: str, project_root: str) -> typing.Union[WorkflowBase, PythonTask]: """ - Load the workflow of a the script file. + Load the workflow of a script file. N.B.: it assumes that the file is self-contained, in other words, there are no relative imports. """ flyte_ctx_builder = context_manager.FlyteContextManager.current_context().new_builder() @@ -675,6 +706,8 @@ def _run(*args, **kwargs): wait=run_level_params.get("wait_execution"), options=options, type_hints=entity.python_interface.inputs, + overwrite_cache=run_level_params.get("overwrite_cache"), + envs=run_level_params.get("envs"), ) console_url = remote.generate_console_url(execution) diff --git a/flytekit/configuration/__init__.py b/flytekit/configuration/__init__.py index 362e6d425d..4108eab231 100644 --- a/flytekit/configuration/__init__.py +++ b/flytekit/configuration/__init__.py @@ -428,7 +428,6 @@ def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None kwargs = set_if_exists(kwargs, "auth_mode", _internal.Credentials.AUTH_MODE.read(config_file)) kwargs = set_if_exists(kwargs, "endpoint", _internal.Platform.URL.read(config_file)) kwargs = set_if_exists(kwargs, "console_endpoint", _internal.Platform.CONSOLE_ENDPOINT.read(config_file)) - kwargs = set_if_exists(kwargs, "http_proxy_url", _internal.Platform.HTTP_PROXY_URL.read(config_file)) return PlatformConfig(**kwargs) diff --git a/flytekit/core/map_task.py b/flytekit/core/map_task.py index 44f18de2a3..6e47ce716c 100644 --- a/flytekit/core/map_task.py +++ b/flytekit/core/map_task.py @@ -16,7 +16,7 @@ from flytekit.core.constants import SdkTaskType from flytekit.core.context_manager import ExecutionState, FlyteContext, FlyteContextManager from flytekit.core.interface import transform_interface_to_list_interface -from flytekit.core.python_function_task import PythonFunctionTask +from flytekit.core.python_function_task import PythonFunctionTask, PythonInstanceTask from flytekit.core.tracker import TrackedInstance from flytekit.core.utils import timeit from flytekit.exceptions import scopes as exception_scopes @@ -34,7 +34,7 @@ class MapPythonTask(PythonTask): def __init__( self, - python_function_task: typing.Union[PythonFunctionTask, functools.partial], + python_function_task: typing.Union[PythonFunctionTask, PythonInstanceTask, functools.partial], concurrency: Optional[int] = None, min_success_ratio: Optional[float] = None, bound_inputs: Optional[Set[str]] = None, @@ -65,7 +65,10 @@ def __init__( actual_task = python_function_task if not isinstance(actual_task, PythonFunctionTask): - raise ValueError("Map tasks can only compose of Python Functon Tasks currently") + if isinstance(actual_task, PythonInstanceTask): + pass + else: + raise ValueError("Map tasks can only compose of PythonFuncton and PythonInstanceTasks currently") if len(actual_task.python_interface.outputs.keys()) > 1: raise ValueError("Map tasks only accept python function tasks with 0 or 1 outputs") @@ -76,7 +79,11 @@ def __init__( collection_interface = transform_interface_to_list_interface(actual_task.python_interface, self._bound_inputs) self._run_task: PythonFunctionTask = actual_task - _, mod, f, _ = tracker.extract_task_module(actual_task.task_function) + if isinstance(actual_task, PythonInstanceTask): + mod = actual_task.task_type + f = actual_task.lhs + else: + _, mod, f, _ = tracker.extract_task_module(actual_task.task_function) h = hashlib.md5(collection_interface.__str__().encode("utf-8")).hexdigest() name = f"{mod}.map_{f}_{h}" @@ -271,7 +278,7 @@ def _raw_execute(self, **kwargs) -> Any: def map_task( - task_function: typing.Union[PythonFunctionTask, functools.partial], + task_function: typing.Union[PythonFunctionTask, PythonInstanceTask, functools.partial], concurrency: int = 0, min_success_ratio: float = 1.0, **kwargs, diff --git a/flytekit/core/node.py b/flytekit/core/node.py index dfb0c65ffb..2bf07c9621 100644 --- a/flytekit/core/node.py +++ b/flytekit/core/node.py @@ -128,6 +128,8 @@ def with_overrides(self, *args, **kwargs): if not isinstance(new_task_config, type(self.flyte_entity._task_config)): raise ValueError("can't change the type of the task config") self.flyte_entity._task_config = new_task_config + if "container_image" in kwargs: + self.flyte_entity._container_image = kwargs["container_image"] return self diff --git a/flytekit/core/promise.py b/flytekit/core/promise.py index 24628cd52e..9da2d7388c 100644 --- a/flytekit/core/promise.py +++ b/flytekit/core/promise.py @@ -613,7 +613,7 @@ def binding_data_from_python_std( f"Cannot pass output from task {t_value.task_name} that produces no outputs to a downstream task" ) - elif expected_literal_type.union_type is not None: + elif t_value is not None and expected_literal_type.union_type is not None: for i in range(len(expected_literal_type.union_type.variants)): try: lt_type = expected_literal_type.union_type.variants[i] diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index da1f614ba1..8376e16ee9 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -173,8 +173,7 @@ def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], exp return self._to_literal_transformer(python_val) def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> T: - if get_origin(expected_python_type) is Annotated: - expected_python_type = get_args(expected_python_type)[0] + expected_python_type = get_underlying_type(expected_python_type) if expected_python_type != self._type: raise TypeTransformerFailedError( @@ -311,7 +310,7 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: Extracts the Literal type definition for a Dataclass and returns a type Struct. If possible also extracts the JSONSchema for the dataclass. """ - if get_origin(t) is Annotated: + if is_annotated(t): raise ValueError( "Flytekit does not currently have support for FlyteAnnotations applied to Dataclass." f"Type {t} cannot be parsed." @@ -368,7 +367,7 @@ def _get_origin_type_in_annotation(self, python_type: Type[T]) -> Type[T]: self._get_origin_type_in_annotation(get_args(python_type)[0]), self._get_origin_type_in_annotation(get_args(python_type)[1]), ] - elif get_origin(python_type) is Annotated: + elif is_annotated(python_type): return get_args(python_type)[0] elif dataclasses.is_dataclass(python_type): for field in dataclasses.fields(copy.deepcopy(python_type)): @@ -725,7 +724,7 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: """ cls.lazy_import_transformers() # Step 1 - if get_origin(python_type) is Annotated: + if is_annotated(python_type): args = get_args(python_type) for annotation in args: if isinstance(annotation, TypeTransformer): @@ -740,7 +739,7 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: if hasattr(python_type, "__origin__"): # Handling of annotated generics, eg: # Annotated[typing.List[int], 'foo'] - if get_origin(python_type) is Annotated: + if is_annotated(python_type): return cls.get_transformer(get_args(python_type)[0]) if python_type.__origin__ in cls._REGISTRY: @@ -811,7 +810,7 @@ def to_literal_type(cls, python_type: Type) -> LiteralType: transformer = cls.get_transformer(python_type) res = transformer.get_literal_type(python_type) data = None - if get_origin(python_type) is Annotated: + if is_annotated(python_type): for x in get_args(python_type)[1:]: if not isinstance(x, FlyteAnnotation): continue @@ -839,9 +838,9 @@ def to_literal(cls, ctx: FlyteContext, python_val: typing.Any, python_type: Type # In case the value is an annotated type we inspect the annotations and look for hash-related annotations. hash = None - if get_origin(python_type) is Annotated: + if is_annotated(python_type): # We are now dealing with one of two cases: - # 1. The annotated type is a `HashMethod`, which indicates that we should we should produce the hash using + # 1. The annotated type is a `HashMethod`, which indicates that we should produce the hash using # the method indicated in the annotation. # 2. The annotated type is being used for a different purpose other than calculating hash values, in which case # we should just continue. @@ -868,7 +867,7 @@ def to_python_value(cls, ctx: FlyteContext, lv: Literal, expected_python_type: T @classmethod def to_html(cls, ctx: FlyteContext, python_val: typing.Any, expected_python_type: Type[typing.Any]) -> str: transformer = cls.get_transformer(expected_python_type) - if get_origin(expected_python_type) is Annotated: + if is_annotated(expected_python_type): expected_python_type, *annotate_args = get_args(expected_python_type) from flytekit.deck.renderer import Renderable @@ -992,7 +991,7 @@ def get_sub_type(t: Type[T]) -> Type[T]: if hasattr(t, "__origin__"): # Handle annotation on list generic, eg: # Annotated[typing.List[int], 'foo'] - if get_origin(t) is Annotated: + if is_annotated(t): return ListTransformer.get_sub_type(get_args(t)[0]) if t.__origin__ is list and hasattr(t, "__args__"): @@ -1018,7 +1017,7 @@ def is_batchable(t: Type): """ from flytekit.types.pickle import FlytePickle - if get_origin(t) is Annotated: + if is_annotated(t): return ListTransformer.is_batchable(get_args(t)[0]) if get_origin(t) is list: subtype = get_args(t)[0] @@ -1035,7 +1034,7 @@ def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], exp batch_size = len(python_val) # default batch size # parse annotated to get the number of items saved in a pickle file. - if get_origin(python_type) is Annotated: + if is_annotated(python_type): for annotation in get_args(python_type)[1:]: if isinstance(annotation, BatchSize): batch_size = annotation.val @@ -1179,8 +1178,7 @@ def get_sub_type_in_optional(t: Type[T]) -> Type[T]: return get_args(t)[0] def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]: - if get_origin(t) is Annotated: - t = get_args(t)[0] + t = get_underlying_type(t) try: trans = [(TypeEngine.get_transformer(x), x) for x in get_args(t)] @@ -1192,8 +1190,7 @@ def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]: raise ValueError(f"Type of Generic Union type is not supported, {e}") def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: - if get_origin(python_type) is Annotated: - python_type = get_args(python_type)[0] + python_type = get_underlying_type(python_type) found_res = False res = None @@ -1218,8 +1215,7 @@ def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], exp raise TypeTransformerFailedError(f"Cannot convert from {python_val} to {python_type}") def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> Optional[typing.Any]: - if get_origin(expected_python_type) is Annotated: - expected_python_type = get_args(expected_python_type)[0] + expected_python_type = get_underlying_type(expected_python_type) union_tag = None union_type = None @@ -1456,7 +1452,7 @@ def __init__(self): super().__init__(name="DefaultEnumTransformer", t=enum.Enum) def get_literal_type(self, t: Type[T]) -> LiteralType: - if get_origin(t) is Annotated: + if is_annotated(t): raise ValueError( f"Flytekit does not currently have support \ for FlyteAnnotations applied to enums. {t} cannot be \ @@ -1765,3 +1761,14 @@ def get(self, attr: str, as_type: Optional[typing.Type] = None) -> typing.Any: _register_default_type_transformers() + + +def is_annotated(t: Type) -> bool: + return get_origin(t) is Annotated + + +def get_underlying_type(t: Type) -> Type: + """Return the underlying type for annotated types or the type itself""" + if is_annotated(t): + return get_args(t)[0] + return t diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index 6a25ab9fc0..c011a218a6 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -4,6 +4,7 @@ from flytekit.core.context_manager import ExecutionParameters, ExecutionState, FlyteContext, FlyteContextManager from flytekit.loggers import logger +from flytekit.tools.interactive import ipython_check OUTPUT_DIR_JUPYTER_PREFIX = "jupyter" DECK_FILE_NAME = "deck.html" @@ -124,22 +125,6 @@ def html(self) -> str: return gantt_chart_html + time_table_html + note -def _ipython_check() -> bool: - """ - Check if interface is launching from iPython (not colab) - :return is_ipython (bool): True or False - """ - is_ipython = False - try: # Check if running interactively using ipython. - from IPython import get_ipython - - if get_ipython() is not None: - is_ipython = True - except (ImportError, NameError): - pass - return is_ipython - - def _get_deck( new_user_params: ExecutionParameters, ignore_jupyter: bool = False ) -> typing.Union[str, "IPython.core.display.HTML"]: # type:ignore @@ -149,7 +134,7 @@ def _get_deck( """ deck_map = {deck.name: deck.html for deck in new_user_params.decks} raw_html = get_deck_template().render(metadata=deck_map) - if not ignore_jupyter and _ipython_check(): + if not ignore_jupyter and ipython_check(): try: from IPython.core.display import HTML except ImportError: diff --git a/flytekit/image_spec/image_spec.py b/flytekit/image_spec/image_spec.py index a0ddbc1977..d01528bfea 100644 --- a/flytekit/image_spec/image_spec.py +++ b/flytekit/image_spec/image_spec.py @@ -32,6 +32,7 @@ class ImageSpec: packages: list of python packages to install. apt_packages: list of apt packages to install. base_image: base image of the image. + platform: Specify the target platforms for the build output (for example, windows/amd64 or linux/amd64,darwin/arm64 """ name: str = "flytekit" @@ -43,6 +44,7 @@ class ImageSpec: packages: Optional[List[str]] = None apt_packages: Optional[List[str]] = None base_image: Optional[str] = None + platform: str = "linux/amd64" def image_name(self) -> str: """ @@ -147,7 +149,7 @@ def calculate_hash_from_image_spec(image_spec: ImageSpec): # copy the image spec to avoid modifying the original image spec. otherwise, the hash will be different. spec = copy(image_spec) spec.source_root = hash_directory(image_spec.source_root) if image_spec.source_root else b"" - image_spec_bytes = bytes(image_spec.to_json(), "utf-8") + image_spec_bytes = bytes(spec.to_json(), "utf-8") tag = base64.urlsafe_b64encode(hashlib.md5(image_spec_bytes).digest()).decode("ascii") # replace "=" with "." to make it a valid tag return tag.replace("=", ".") diff --git a/flytekit/models/common.py b/flytekit/models/common.py index 4f030e25a4..79392700e2 100644 --- a/flytekit/models/common.py +++ b/flytekit/models/common.py @@ -1,8 +1,10 @@ import abc as _abc import json as _json import re +from typing import Dict from flyteidl.admin import common_pb2 as _common_pb2 +from flyteidl.core import literals_pb2 as _literals_pb2 from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct @@ -485,3 +487,19 @@ def to_flyte_idl(self): @classmethod def from_flyte_idl(cls, pb2): return cls(output_location_prefix=pb2.output_location_prefix) + + +class Envs(FlyteIdlEntity): + def __init__(self, envs: Dict[str, str]): + self._envs = envs + + @property + def envs(self) -> Dict[str, str]: + return self._envs + + def to_flyte_idl(self) -> _common_pb2.Envs: + return _common_pb2.Envs(values=[_literals_pb2.KeyValuePair(key=k, value=v) for k, v in self.envs.items()]) + + @classmethod + def from_flyte_idl(cls, pb2: _common_pb2.Envs) -> _common_pb2.Envs: + return cls(envs={kv.key: kv.value for kv in pb2.values}) diff --git a/flytekit/models/execution.py b/flytekit/models/execution.py index 9c2d5ba2ec..23b4baab01 100644 --- a/flytekit/models/execution.py +++ b/flytekit/models/execution.py @@ -2,6 +2,7 @@ import datetime import typing +from typing import Optional import flyteidl import flyteidl.admin.execution_pb2 as _execution_pb2 @@ -47,10 +48,10 @@ def __init__( mode: int, principal: str, nesting: int, - scheduled_at: typing.Optional[datetime.datetime] = None, - parent_node_execution: typing.Optional[_identifier.NodeExecutionIdentifier] = None, - reference_execution: typing.Optional[_identifier.WorkflowExecutionIdentifier] = None, - system_metadata: typing.Optional[SystemMetadata] = None, + scheduled_at: Optional[datetime.datetime] = None, + parent_node_execution: Optional[_identifier.NodeExecutionIdentifier] = None, + reference_execution: Optional[_identifier.WorkflowExecutionIdentifier] = None, + system_metadata: Optional[SystemMetadata] = None, ): """ :param mode: An enum value from ExecutionMetadata.ExecutionMode which specifies how the job started. @@ -173,9 +174,10 @@ def __init__( annotations=None, auth_role=None, raw_output_data_config=None, - max_parallelism=None, - security_context: typing.Optional[security.SecurityContext] = None, - overwrite_cache: bool = None, + max_parallelism: Optional[int] = None, + security_context: Optional[security.SecurityContext] = None, + overwrite_cache: Optional[bool] = None, + envs: Optional[_common_models.Envs] = None, ): """ :param flytekit.models.core.identifier.Identifier launch_plan: Launch plan unique identifier to execute @@ -189,7 +191,9 @@ def __init__( :param max_parallelism int: Controls the maximum number of tasknodes that can be run in parallel for the entire workflow. This is useful to achieve fairness. Note: MapTasks are regarded as one unit, and parallelism/concurrency of MapTasks is independent from this. - + :param security_context: Optional security context to use for this execution. + :param overwrite_cache: Optional flag to overwrite the cache for this execution. + :param envs: flytekit.models.common.Envs environment variables to set for this execution. """ self._launch_plan = launch_plan self._metadata = metadata @@ -201,7 +205,8 @@ def __init__( self._raw_output_data_config = raw_output_data_config self._max_parallelism = max_parallelism self._security_context = security_context - self.overwrite_cache = overwrite_cache + self._overwrite_cache = overwrite_cache + self._envs = envs @property def launch_plan(self): @@ -268,6 +273,14 @@ def max_parallelism(self) -> int: def security_context(self) -> typing.Optional[security.SecurityContext]: return self._security_context + @property + def overwrite_cache(self) -> Optional[bool]: + return self._overwrite_cache + + @property + def envs(self) -> Optional[_common_models.Envs]: + return self._envs + def to_flyte_idl(self): """ :rtype: flyteidl.admin.execution_pb2.ExecutionSpec @@ -276,7 +289,7 @@ def to_flyte_idl(self): launch_plan=self.launch_plan.to_flyte_idl(), metadata=self.metadata.to_flyte_idl(), notifications=self.notifications.to_flyte_idl() if self.notifications else None, - disable_all=self.disable_all, + disable_all=self.disable_all, # type: ignore labels=self.labels.to_flyte_idl(), annotations=self.annotations.to_flyte_idl(), auth_role=self._auth_role.to_flyte_idl() if self.auth_role else None, @@ -286,6 +299,7 @@ def to_flyte_idl(self): max_parallelism=self.max_parallelism, security_context=self.security_context.to_flyte_idl() if self.security_context else None, overwrite_cache=self.overwrite_cache, + envs=self.envs.to_flyte_idl() if self.envs else None, ) @classmethod @@ -310,6 +324,7 @@ def from_flyte_idl(cls, p): if p.security_context else None, overwrite_cache=p.overwrite_cache, + envs=_common_models.Envs.from_flyte_idl(p.envs) if p.HasField("envs") else None, ) diff --git a/flytekit/remote/entities.py b/flytekit/remote/entities.py index 624e01661d..fcca6cf151 100644 --- a/flytekit/remote/entities.py +++ b/flytekit/remote/entities.py @@ -362,6 +362,7 @@ def __init__( ) # TODO: Revisit flyte_branch_node and flyte_gate_node, should they be another type like Condition instead # of a node? + self._flyte_task_node = task_node if task_node: self._flyte_entity = task_node.flyte_task elif workflow_node: @@ -382,6 +383,10 @@ def __init__( ) self._upstream = upstream_nodes + @property + def task_node(self) -> Optional[FlyteTaskNode]: + return self._flyte_task_node + @property def flyte_entity(self) -> Union[FlyteTask, FlyteWorkflow, FlyteLaunchPlan, FlyteBranchNode]: return self._flyte_entity @@ -707,7 +712,7 @@ def promote_from_closure( :param closure: This is the closure returned by Admin :param node_launch_plans: The reason this exists is because the compiled closure doesn't have launch plans. - It only has subworkflows and tasks. Why this is is unclear. If supplied, this map of launch plans will be + It only has subworkflows and tasks. Why this is unclear. If supplied, this map of launch plans will be :return: """ sub_workflows = {sw.template.id: sw.template for sw in closure.sub_workflows} diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 91189ede74..e0a411de50 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -22,7 +22,6 @@ from flyteidl.admin.signal_pb2 import Signal, SignalListRequest, SignalSetRequest from flyteidl.core import literals_pb2 as literals_pb2 -from flytekit import Literal from flytekit.clients.friendly import SynchronousFlyteClient from flytekit.clients.helpers import iterate_node_executions, iterate_task_executions from flytekit.configuration import Config, FastSerializationSettings, ImageConfig, SerializationSettings @@ -61,6 +60,7 @@ NotificationList, WorkflowExecutionGetDataResponse, ) +from flytekit.models.literals import Literal, LiteralMap from flytekit.remote.backfill import create_backfill_workflow from flytekit.remote.entities import FlyteLaunchPlan, FlyteNode, FlyteTask, FlyteTaskNode, FlyteWorkflow from flytekit.remote.executions import FlyteNodeExecution, FlyteTaskExecution, FlyteWorkflowExecution @@ -68,6 +68,7 @@ from flytekit.remote.lazy_entity import LazyEntity from flytekit.remote.remote_callable import RemoteEntity from flytekit.tools.fast_registration import fast_package +from flytekit.tools.interactive import ipython_check from flytekit.tools.script_mode import compress_scripts, hash_file from flytekit.tools.translator import ( FlyteControlPlaneEntity, @@ -77,6 +78,11 @@ get_serializable_launch_plan, ) +try: + from IPython.core.display import HTML +except ImportError: + ... + ExecutionDataResponse = typing.Union[WorkflowExecutionGetDataResponse, NodeExecutionGetDataResponse] MOST_RECENT_FIRST = admin_common_models.Sort("created_at", admin_common_models.Sort.Direction.DESCENDING) @@ -219,6 +225,48 @@ def file_access(self) -> FileAccessProvider: """File access provider to use for offloading non-literal inputs/outputs.""" return self._file_access + def get( + self, flyte_uri: typing.Optional[str] = None + ) -> typing.Optional[typing.Union[LiteralsResolver, Literal, HTML, bytes]]: + """ + General function that works with flyte tiny urls. This can return outputs (in the form of LiteralsResolver, or + individual Literals for singular requests), or HTML if passed a deck link, or bytes containing HTML, + if ipython is not available locally. + """ + if flyte_uri is None: + raise user_exceptions.FlyteUserException("flyte_uri cannot be empty") + ctx = self._ctx or FlyteContextManager.current_context() + try: + data_response = self.client.get_data(flyte_uri) + + if data_response.HasField("literal_map"): + lm = LiteralMap.from_flyte_idl(data_response.literal_map) + return LiteralsResolver(lm.literals) + elif data_response.HasField("literal"): + return data_response.literal + elif data_response.HasField("pre_signed_urls"): + if len(data_response.pre_signed_urls.signed_url) == 0: + raise ValueError(f"Flyte url {flyte_uri} resolved to empty download link") + d = data_response.pre_signed_urls.signed_url[0] + remote_logger.debug(f"Download link is {d}") + fs = ctx.file_access.get_filesystem_for_path(d) + + # If the venv has IPython, then return IPython's HTML + if ipython_check(): + remote_logger.debug(f"IPython found, returning HTML from {flyte_uri}") + with fs.open(d, "rb") as r: + html = HTML(str(r.read())) + return html + # If not return bytes + else: + remote_logger.debug(f"IPython not found, returning HTML as bytes from {flyte_uri}") + return fs.open(d, "rb").read() + + except user_exceptions.FlyteUserException as e: + remote_logger.info(f"Error from Flyte backend when trying to fetch data: {e.__cause__}") + + remote_logger.info(f"Nothing found from {flyte_uri}") + def remote_context(self): """Context manager with remote-specific configuration.""" return FlyteContextManager.with_context( @@ -747,6 +795,9 @@ def upload_file( upload_location.signed_url, data=content, headers={"Content-Length": str(content_length), "Content-MD5": encoded_md5}, + verify=False + if self._config.platform.insecure_skip_verify is True + else self._config.platform.ca_cert_file_path, ) if rsp.status_code != requests.codes["OK"]: @@ -899,7 +950,8 @@ def _execute( options: typing.Optional[Options] = None, wait: bool = False, type_hints: typing.Optional[typing.Dict[str, typing.Type]] = None, - overwrite_cache: bool = None, + overwrite_cache: typing.Optional[bool] = None, + envs: typing.Optional[typing.Dict[str, str]] = None, ) -> FlyteWorkflowExecution: """Common method for execution across all entities. @@ -914,6 +966,7 @@ def _execute( :param overwrite_cache: Allows for all cached values of a workflow and its tasks to be overwritten for a single execution. If enabled, all calculations are performed even if cached results would be available, overwriting the stored data once execution finishes successfully. + :param envs: Environment variables to set for the execution. :returns: :class:`~flytekit.remote.workflow_execution.FlyteWorkflowExecution` """ execution_name = execution_name or "f" + uuid.uuid4().hex[:19] @@ -978,6 +1031,7 @@ def _execute( auth_role=None, max_parallelism=options.max_parallelism, security_context=options.security_context, + envs=common_models.Envs(envs) if envs else None, ), literal_inputs, ) @@ -1033,7 +1087,8 @@ def execute( options: typing.Optional[Options] = None, wait: bool = False, type_hints: typing.Optional[typing.Dict[str, typing.Type]] = None, - overwrite_cache: bool = None, + overwrite_cache: typing.Optional[bool] = None, + envs: typing.Optional[typing.Dict[str, str]] = None, ) -> FlyteWorkflowExecution: """ Execute a task, workflow, or launchplan, either something that's been declared locally, or a fetched entity. @@ -1070,6 +1125,7 @@ def execute( :param overwrite_cache: Allows for all cached values of a workflow and its tasks to be overwritten for a single execution. If enabled, all calculations are performed even if cached results would be available, overwriting the stored data once execution finishes successfully. + :param envs: Environment variables to be set for the execution. .. note: @@ -1089,6 +1145,7 @@ def execute( wait=wait, type_hints=type_hints, overwrite_cache=overwrite_cache, + envs=envs, ) if isinstance(entity, FlyteWorkflow): return self.execute_remote_wf( @@ -1101,6 +1158,7 @@ def execute( wait=wait, type_hints=type_hints, overwrite_cache=overwrite_cache, + envs=envs, ) if isinstance(entity, PythonTask): return self.execute_local_task( @@ -1114,6 +1172,7 @@ def execute( image_config=image_config, wait=wait, overwrite_cache=overwrite_cache, + envs=envs, ) if isinstance(entity, WorkflowBase): return self.execute_local_workflow( @@ -1128,6 +1187,7 @@ def execute( options=options, wait=wait, overwrite_cache=overwrite_cache, + envs=envs, ) if isinstance(entity, LaunchPlan): return self.execute_local_launch_plan( @@ -1140,6 +1200,7 @@ def execute( options=options, wait=wait, overwrite_cache=overwrite_cache, + envs=envs, ) raise NotImplementedError(f"entity type {type(entity)} not recognized for execution") @@ -1156,7 +1217,8 @@ def execute_remote_task_lp( options: typing.Optional[Options] = None, wait: bool = False, type_hints: typing.Optional[typing.Dict[str, typing.Type]] = None, - overwrite_cache: bool = None, + overwrite_cache: typing.Optional[bool] = None, + envs: typing.Optional[typing.Dict[str, str]] = None, ) -> FlyteWorkflowExecution: """Execute a FlyteTask, or FlyteLaunchplan. @@ -1172,6 +1234,7 @@ def execute_remote_task_lp( options=options, type_hints=type_hints, overwrite_cache=overwrite_cache, + envs=envs, ) def execute_remote_wf( @@ -1184,7 +1247,8 @@ def execute_remote_wf( options: typing.Optional[Options] = None, wait: bool = False, type_hints: typing.Optional[typing.Dict[str, typing.Type]] = None, - overwrite_cache: bool = None, + overwrite_cache: typing.Optional[bool] = None, + envs: typing.Optional[typing.Dict[str, str]] = None, ) -> FlyteWorkflowExecution: """Execute a FlyteWorkflow. @@ -1201,6 +1265,7 @@ def execute_remote_wf( wait=wait, type_hints=type_hints, overwrite_cache=overwrite_cache, + envs=envs, ) # Flytekit Entities @@ -1217,23 +1282,24 @@ def execute_local_task( execution_name: str = None, image_config: typing.Optional[ImageConfig] = None, wait: bool = False, - overwrite_cache: bool = None, + overwrite_cache: typing.Optional[bool] = None, + envs: typing.Optional[typing.Dict[str, str]] = None, ) -> FlyteWorkflowExecution: """ - Execute an @task-decorated function or TaskTemplate task. + Execute a @task-decorated function or TaskTemplate task. - :param entity: - :param inputs: - register the task, which requires compiling the task, before running it. - :param project: - :param domain: - :param name: - :param version: - :param execution_name: - :param image_config: - :param wait: - :param overwrite_cache: - :return: + :param entity: local task entity. + :param inputs: register the task, which requires compiling the task, before running it. + :param project: The execution project, will default to the Remote's default project. + :param domain: The execution domain, will default to the Remote's default domain. + :param name: specific name of the task to run. + :param version: specific version of the task to run. + :param execution_name: If provided, will use this name for the execution. + :param image_config: If provided, will use this image config in the pod. + :param wait: If True, will wait for the execution to complete before returning. + :param overwrite_cache: If True, will overwrite the cache. + :param envs: Environment variables to set for the execution. + :return: FlyteWorkflowExecution object. """ resolved_identifiers = self._resolve_identifier_kwargs(entity, project, domain, name, version) resolved_identifiers_dict = asdict(resolved_identifiers) @@ -1260,6 +1326,7 @@ def execute_local_task( wait=wait, type_hints=entity.python_interface.inputs, overwrite_cache=overwrite_cache, + envs=envs, ) def execute_local_workflow( @@ -1274,7 +1341,8 @@ def execute_local_workflow( image_config: typing.Optional[ImageConfig] = None, options: typing.Optional[Options] = None, wait: bool = False, - overwrite_cache: bool = None, + overwrite_cache: typing.Optional[bool] = None, + envs: typing.Optional[typing.Dict[str, str]] = None, ) -> FlyteWorkflowExecution: """ Execute an @workflow decorated function. @@ -1289,6 +1357,7 @@ def execute_local_workflow( :param options: :param wait: :param overwrite_cache: + :param envs: :return: """ resolved_identifiers = self._resolve_identifier_kwargs(entity, project, domain, name, version) @@ -1334,6 +1403,7 @@ def execute_local_workflow( options=options, type_hints=entity.python_interface.inputs, overwrite_cache=overwrite_cache, + envs=envs, ) def execute_local_launch_plan( @@ -1346,7 +1416,8 @@ def execute_local_launch_plan( execution_name: typing.Optional[str] = None, options: typing.Optional[Options] = None, wait: bool = False, - overwrite_cache: bool = None, + overwrite_cache: typing.Optional[bool] = None, + envs: typing.Optional[typing.Dict[str, str]] = None, ) -> FlyteWorkflowExecution: """ @@ -1356,10 +1427,11 @@ def execute_local_launch_plan( :param project: The same as version, but will default to the Remote object's project :param domain: The same as version, but will default to the Remote object's domain :param execution_name: If specified, will be used as the execution name instead of randomly generating. - :param options: - :param wait: - :param overwrite_cache: - :return: + :param options: Options to be passed into the execution. + :param wait: If True, will wait for the execution to complete before returning. + :param overwrite_cache: If True, will overwrite the cache. + :param envs: Environment variables to be passed into the execution. + :return: FlyteWorkflowExecution object """ try: flyte_launchplan: FlyteLaunchPlan = self.fetch_launch_plan( @@ -1385,6 +1457,7 @@ def execute_local_launch_plan( wait=wait, type_hints=entity.python_interface.inputs, overwrite_cache=overwrite_cache, + envs=envs, ) ################################### @@ -1610,13 +1683,10 @@ def sync_node_execution( self.sync_node_execution(FlyteNodeExecution.promote_from_model(cne), dynamic_flyte_wf._node_map) for cne in child_node_executions ] - # This is copied from below - dynamic tasks have both task executions (executions of the parent - # task) as well as underlying node executions (of the generated subworkflow). Feel free to refactor - # if you can think of a better way. execution._task_executions = [ - self.sync_task_execution(FlyteTaskExecution.promote_from_model(t)) - for t in iterate_task_executions(self.client, execution.id) + node_exes.task_executions for node_exes in execution.subworkflow_node_executions.values() ] + execution._interface = dynamic_flyte_wf.interface # Handle the case where it's a static subworkflow @@ -1643,7 +1713,9 @@ def sync_node_execution( # This is the plain ol' task execution case else: execution._task_executions = [ - self.sync_task_execution(FlyteTaskExecution.promote_from_model(t)) + self.sync_task_execution( + FlyteTaskExecution.promote_from_model(t), node_mapping[node_id].task_node.flyte_task + ) for t in iterate_task_executions(self.client, execution.id) ] execution._interface = execution._node.flyte_entity.interface @@ -1657,17 +1729,15 @@ def sync_node_execution( return execution def sync_task_execution( - self, execution: FlyteTaskExecution, entity_definition: typing.Union[FlyteWorkflow, FlyteTask] = None + self, execution: FlyteTaskExecution, entity_definition: typing.Optional[FlyteTask] = None ) -> FlyteTaskExecution: """Sync a FlyteTaskExecution object with its corresponding remote state.""" - if entity_definition is not None: - raise ValueError("Entity definition arguments aren't supported when syncing task executions") - execution._closure = self.client.get_task_execution(execution.id).closure execution_data = self.client.get_task_execution_data(execution.id) task_id = execution.id.task_id - task = self.fetch_task(task_id.project, task_id.domain, task_id.name, task_id.version) - return self._assign_inputs_and_outputs(execution, execution_data, task.interface) + if entity_definition is None: + entity_definition = self.fetch_task(task_id.project, task_id.domain, task_id.name, task_id.version) + return self._assign_inputs_and_outputs(execution, execution_data, entity_definition.interface) ############################# # Terminate Execution State # diff --git a/flytekit/tools/interactive.py b/flytekit/tools/interactive.py new file mode 100644 index 0000000000..a0d022df18 --- /dev/null +++ b/flytekit/tools/interactive.py @@ -0,0 +1,14 @@ +def ipython_check() -> bool: + """ + Check if interface is launching from iPython (not colab) + :return is_ipython (bool): True or False + """ + is_ipython = False + try: # Check if running interactively using ipython. + from IPython import get_ipython + + if get_ipython() is not None: + is_ipython = True + except (ImportError, NameError): + pass + return is_ipython diff --git a/flytekit/tools/repo.py b/flytekit/tools/repo.py index 82d4c2c226..e55350d3ae 100644 --- a/flytekit/tools/repo.py +++ b/flytekit/tools/repo.py @@ -254,14 +254,14 @@ def register( options = Options.default_from(k8s_service_account=service_account, raw_data_prefix=raw_data_prefix) # Load all the entities - serializable_entities = load_packages_and_modules( + registrable_entities = load_packages_and_modules( serialization_settings, detected_root, list(package_or_module), options ) - if len(serializable_entities) == 0: + if len(registrable_entities) == 0: click.secho("No Flyte entities were detected. Aborting!", fg="red") return - for cp_entity in serializable_entities: + for cp_entity in registrable_entities: og_id = cp_entity.id if isinstance(cp_entity, launch_plan.LaunchPlan) else cp_entity.template.id secho(og_id, "") try: @@ -274,4 +274,4 @@ def register( secho(og_id, reason="Dry run Mode!") except RegistrationSkipped: secho(og_id, "failed") - click.secho(f"Successfully registered {len(serializable_entities)} entities", fg="green") + click.secho(f"Successfully registered {len(registrable_entities)} entities", fg="green") diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 9508dee2e2..42073e6f53 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -8,10 +8,9 @@ from dataclasses_json import config, dataclass_json from marshmallow import fields -from typing_extensions import Annotated, get_args, get_origin from flytekit.core.context_manager import FlyteContext, FlyteContextManager -from flytekit.core.type_engine import TypeEngine, TypeTransformer, TypeTransformerFailedError +from flytekit.core.type_engine import TypeEngine, TypeTransformer, TypeTransformerFailedError, get_underlying_type from flytekit.loggers import logger from flytekit.models.core.types import BlobType from flytekit.models.literals import Blob, BlobMetadata, Literal, Scalar @@ -337,8 +336,7 @@ def to_literal( raise TypeTransformerFailedError("None value cannot be converted to a file.") # Correctly handle `Annotated[FlyteFile, ...]` by extracting the origin type - if get_origin(python_type) is Annotated: - python_type = get_args(python_type)[0] + python_type = get_underlying_type(python_type) if not (python_type is os.PathLike or issubclass(python_type, FlyteFile)): raise ValueError(f"Incorrect type {python_type}, must be either a FlyteFile or os.PathLike") @@ -413,6 +411,9 @@ def to_python_value( if expected_python_type is os.PathLike: return FlyteFile(uri) + # Correctly handle `Annotated[FlyteFile, ...]` by extracting the origin type + expected_python_type = get_underlying_type(expected_python_type) + # The rest of the logic is only for FlyteFile types. if not issubclass(expected_python_type, FlyteFile): raise TypeError(f"Neither os.PathLike nor FlyteFile specified {expected_python_type}") diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt index 30900d53ff..2e74440f88 100644 --- a/plugins/flytekit-aws-athena/requirements.txt +++ b/plugins/flytekit-aws-athena/requirements.txt @@ -6,53 +6,141 @@ # -e file:.#egg=flytekitplugins-athena # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 - # via retry -deprecated==1.2.13 + # via gcsfs +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-athena +flytekit==1.2.14 # via flytekitplugins-athena -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -61,14 +149,20 @@ grpcio==1.48.2 grpcio-status==1.48.2 # via flytekit idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit # keyring + # rich-click importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jeepney==0.8.0 @@ -76,16 +170,18 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 + # via flytekit +keyring==24.1.1 # via flytekit -keyring==23.13.1 +kubernetes==26.1.0 # via flytekit -markupsafe==2.1.2 +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -96,53 +192,81 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit numpy==1.21.6 # via # flytekit # pandas # pyarrow -packaging==23.0 +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 # via # docker # marshmallow pandas==1.3.5 # via flytekit +portalocker==2.7.0 + # via msal-extensions protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -150,17 +274,39 @@ pyyaml==6.0 # via # cookiecutter # flytekit -regex==2022.10.31 + # kubernetes + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib # responses -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.1.0 # via flytekit secretstorage==3.3.3 # via keyring @@ -168,42 +314,68 @@ singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -toml==0.10.2 - # via responses -types-toml==0.10.8.5 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # importlib-metadata + # markdown-it-py + # pyjwt # responses + # rich # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/plugins/flytekit-aws-athena/setup.py b/plugins/flytekit-aws-athena/setup.py index 9c09d9f921..a2a2457f1f 100644 --- a/plugins/flytekit-aws-athena/setup.py +++ b/plugins/flytekit-aws-athena/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-aws-batch/requirements.txt b/plugins/flytekit-aws-batch/requirements.txt index 7fa8195e20..2cd837184c 100644 --- a/plugins/flytekit-aws-batch/requirements.txt +++ b/plugins/flytekit-aws-batch/requirements.txt @@ -6,53 +6,141 @@ # -e file:.#egg=flytekitplugins-awsbatch # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 - # via retry -deprecated==1.2.13 + # via gcsfs +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-awsbatch +flytekit==1.2.14 # via flytekitplugins-awsbatch -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -61,14 +149,20 @@ grpcio==1.48.2 grpcio-status==1.48.2 # via flytekit idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit # keyring + # rich-click importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jeepney==0.8.0 @@ -76,16 +170,18 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 + # via flytekit +keyring==24.1.1 # via flytekit -keyring==23.13.1 +kubernetes==26.1.0 # via flytekit -markupsafe==2.1.2 +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -96,53 +192,81 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit numpy==1.21.6 # via # flytekit # pandas # pyarrow -packaging==23.0 +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 # via # docker # marshmallow pandas==1.3.5 # via flytekit +portalocker==2.7.0 + # via msal-extensions protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -150,17 +274,39 @@ pyyaml==6.0 # via # cookiecutter # flytekit -regex==2022.10.31 + # kubernetes + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib # responses -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.1.0 # via flytekit secretstorage==3.3.3 # via keyring @@ -168,42 +314,68 @@ singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -toml==0.10.2 - # via responses -types-toml==0.10.8.5 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # importlib-metadata + # markdown-it-py + # pyjwt # responses + # rich # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/plugins/flytekit-aws-batch/setup.py b/plugins/flytekit-aws-batch/setup.py index 0010c19961..79d460ea8c 100644 --- a/plugins/flytekit-aws-batch/setup.py +++ b/plugins/flytekit-aws-batch/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt index 37771df5ce..c49c5a6ba7 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.txt +++ b/plugins/flytekit-aws-sagemaker/requirements.txt @@ -47,6 +47,7 @@ dataclasses-json==0.5.7 # via flytekit decorator==5.1.1 # via + # retry # retry2 deprecated==1.2.13 # via flytekit @@ -58,8 +59,10 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.9 - # via flytekit +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-awssagemaker flytekit==1.2.7 # via flytekitplugins-awssagemaker gevent==22.10.2 @@ -153,6 +156,8 @@ protoc-gen-swagger==0.1.0 # via flyteidl psutil==5.9.4 # via sagemaker-training +py==1.11.0 + # via retry pyarrow==10.0.1 # via flytekit pycparser==2.21 @@ -192,6 +197,8 @@ requests==2.28.2 # responses responses==0.22.0 # via flytekit +retry==0.9.2 + # via flytekit retry2==0.9.5 # via flytekitplugins-awssagemaker retrying==1.3.4 diff --git a/plugins/flytekit-aws-sagemaker/setup.py b/plugins/flytekit-aws-sagemaker/setup.py index ade15df0e8..f95aebfd75 100644 --- a/plugins/flytekit-aws-sagemaker/setup.py +++ b/plugins/flytekit-aws-sagemaker/setup.py @@ -4,7 +4,12 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "sagemaker-training>=3.6.2,<4.0.0", "retry2==0.9.5"] +plugin_requires = [ + "flytekit>=1.1.0b0,<1.3.0,<2.0.0", + "sagemaker-training>=3.6.2,<4.0.0", + "retry2==0.9.5", + "flyteidl>=1.2.11,<1.3.0", +] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-bigquery/requirements.txt b/plugins/flytekit-bigquery/requirements.txt index c89afe3cc6..94b87c13fa 100644 --- a/plugins/flytekit-bigquery/requirements.txt +++ b/plugins/flytekit-bigquery/requirements.txt @@ -6,71 +6,147 @@ # -e file:.#egg=flytekitplugins-bigquery # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -cachetools==5.3.0 +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 # via google-auth -certifi==2022.12.7 - # via requests +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 - # via retry -deprecated==1.2.13 + # via gcsfs +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.10 - # via flytekit -flytekit==1.2.9 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-bigquery +flytekit==1.2.14 # via flytekitplugins-bigquery -google-api-core[grpc]==2.11.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core[grpc]==2.11.1 # via # google-cloud-bigquery # google-cloud-core -google-auth==2.16.1 + # google-cloud-storage +google-auth==2.21.0 # via + # gcsfs # google-api-core + # google-auth-oauthlib # google-cloud-core -google-cloud-bigquery==3.6.0 + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-bigquery==3.11.3 # via flytekitplugins-bigquery -google-cloud-core==2.3.2 - # via google-cloud-bigquery +google-cloud-core==2.3.3 + # via + # google-cloud-bigquery + # google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.4.1 - # via google-cloud-bigquery -googleapis-common-protos==1.58.0 +google-resumable-media==2.5.0 + # via + # google-cloud-bigquery + # google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit # google-api-core # grpcio-status grpcio==1.48.2 @@ -84,14 +160,20 @@ grpcio-status==1.48.2 # flytekit # google-api-core idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit # keyring + # rich-click importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jeepney==0.8.0 @@ -99,16 +181,18 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 # via flytekit -keyring==23.13.1 +keyring==24.1.1 # via flytekit -markupsafe==2.1.2 +kubernetes==26.1.0 + # via flytekit +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -119,25 +203,42 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit numpy==1.21.6 # via # flytekit # pandas # pyarrow -packaging==23.0 +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 # via # docker # google-cloud-bigquery # marshmallow pandas==1.3.5 # via flytekit -proto-plus==1.22.2 +portalocker==2.7.0 + # via msal-extensions +proto-plus==1.22.3 # via google-cloud-bigquery protobuf==3.20.3 # via @@ -151,34 +252,38 @@ protobuf==3.20.3 # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit -pyasn1==0.4.8 +pyasn1==0.5.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 +pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit # google-cloud-bigquery + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -186,65 +291,109 @@ pyyaml==6.0 # via # cookiecutter # flytekit -regex==2022.10.31 + # kubernetes + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs # google-api-core # google-cloud-bigquery + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib # responses -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 # via flytekit rsa==4.9 # via google-auth +s3fs==2023.1.0 + # via flytekit secretstorage==3.3.3 # via keyring singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity # google-auth # grpcio + # isodate + # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -toml==0.10.2 - # via responses -types-toml==0.10.8.5 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # importlib-metadata + # markdown-it-py + # pyjwt # responses + # rich # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/plugins/flytekit-bigquery/setup.py b/plugins/flytekit-bigquery/setup.py index 4bb161301a..96d62b54e2 100644 --- a/plugins/flytekit-bigquery/setup.py +++ b/plugins/flytekit-bigquery/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "google-cloud-bigquery", "flyteidl>=1.2.10,<1.3.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "google-cloud-bigquery", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt index 74ab73760d..0bab09fd21 100644 --- a/plugins/flytekit-data-fsspec/requirements.txt +++ b/plugins/flytekit-data-fsspec/requirements.txt @@ -6,214 +6,14 @@ # -e file:.#egg=flytekitplugins-data-fsspec # via -r requirements.in -arrow==1.2.3 - # via jinja2-time -binaryornot==0.4.4 - # via cookiecutter -botocore==1.29.77 +flyteidl==1.2.11 # via flytekitplugins-data-fsspec -certifi==2022.12.7 - # via requests -cffi==1.15.1 - # via cryptography -chardet==5.1.0 - # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 - # via - # cookiecutter - # flytekit -cloudpickle==2.2.1 - # via flytekit -cookiecutter==2.1.1 - # via flytekit -croniter==1.3.8 - # via flytekit -cryptography==39.0.1 - # via - # pyopenssl - # secretstorage -dataclasses-json==0.5.7 - # via flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker==6.0.1 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.15 - # via flytekit -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 - # via flytekitplugins-data-fsspec -fsspec==2023.1.0 - # via flytekitplugins-data-fsspec -googleapis-common-protos==1.58.0 - # via - # flyteidl - # grpcio-status -grpcio==1.48.2 - # via - # flytekit - # grpcio-status -grpcio-status==1.48.2 - # via flytekit -idna==3.4 - # via requests -importlib-metadata==6.0.0 - # via - # click - # flytekit - # keyring -importlib-resources==5.12.0 - # via keyring -jaraco-classes==3.2.3 - # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -jmespath==1.0.1 - # via botocore -joblib==1.2.0 - # via flytekit -keyring==23.13.1 - # via flytekit -markupsafe==2.1.2 - # via jinja2 -marshmallow==3.19.0 - # via - # dataclasses-json - # marshmallow-enum - # marshmallow-jsonschema -marshmallow-enum==1.5.1 - # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit -more-itertools==9.0.0 - # via jaraco-classes -mypy-extensions==1.0.0 - # via typing-inspect -natsort==8.2.0 - # via flytekit -numpy==1.21.6 - # via - # flytekit - # pandas - # pyarrow -packaging==23.0 - # via - # docker - # marshmallow -pandas==1.3.5 - # via - # flytekit - # flytekitplugins-data-fsspec +googleapis-common-protos==1.59.1 + # via flyteidl protobuf==3.20.3 # via # flyteidl - # flytekit # googleapis-common-protos - # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry -pyarrow==10.0.1 - # via flytekit -pycparser==2.21 - # via cffi -pyopenssl==23.0.0 - # via flytekit -python-dateutil==2.8.2 - # via - # arrow - # botocore - # croniter - # flytekit - # pandas -python-json-logger==2.0.7 - # via flytekit -python-slugify==8.0.0 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit -pytz==2022.7.1 - # via - # flytekit - # pandas -pyyaml==6.0 - # via - # cookiecutter - # flytekit -regex==2022.10.31 - # via docker-image-py -requests==2.28.2 - # via - # cookiecutter - # docker - # flytekit - # responses -responses==0.22.0 - # via flytekit -retry==0.9.2 - # via flytekit -secretstorage==3.3.3 - # via keyring -singledispatchmethod==1.0 - # via flytekit -six==1.16.0 - # via - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -toml==0.10.2 - # via responses -types-toml==0.10.8.5 - # via responses -typing-extensions==4.5.0 - # via - # arrow - # flytekit - # importlib-metadata - # responses - # typing-inspect -typing-inspect==0.8.0 - # via dataclasses-json -urllib3==1.26.14 - # via - # botocore - # docker - # flytekit - # requests - # responses -websocket-client==1.5.1 - # via docker -wheel==0.38.4 - # via flytekit -wrapt==1.14.1 - # via - # deprecated - # flytekit -zipp==3.14.0 - # via - # importlib-metadata - # importlib-resources diff --git a/plugins/flytekit-data-fsspec/setup.py b/plugins/flytekit-data-fsspec/setup.py index 7102e4fb5f..3b4da861ab 100644 --- a/plugins/flytekit-data-fsspec/setup.py +++ b/plugins/flytekit-data-fsspec/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-data-{PLUGIN_NAME}" -plugin_requires = [] +plugin_requires = ["flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-dbt/requirements.txt b/plugins/flytekit-dbt/requirements.txt index 6001a4f957..12bf98cf5e 100644 --- a/plugins/flytekit-dbt/requirements.txt +++ b/plugins/flytekit-dbt/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile requirements.in # @@ -66,8 +66,10 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==1.2.9 - # via flytekit +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-dbt flytekit==1.2.7 # via flytekitplugins-dbt future==0.18.2 @@ -108,6 +110,8 @@ jinja2==2.11.3 # jinja2-time jinja2-time==0.2.0 # via cookiecutter +joblib==1.3.1 + # via flytekit jsonschema==3.1.1 # via hologram keyring==23.5.0 @@ -143,6 +147,7 @@ networkx==2.7.1 # via dbt-core numpy==1.22.3 # via + # flytekit # pandas # pyarrow packaging==21.3 diff --git a/plugins/flytekit-dbt/setup.py b/plugins/flytekit-dbt/setup.py index a2405df6b9..a52be38486 100644 --- a/plugins/flytekit-dbt/setup.py +++ b/plugins/flytekit-dbt/setup.py @@ -7,6 +7,7 @@ plugin_requires = [ "flytekit>=1.1.0,<1.3.0,<2.0.0", "dbt-core>=1.0.0", + "flyteidl>=1.2.11,<1.3.0", ] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-deck-standard/requirements.txt b/plugins/flytekit-deck-standard/requirements.txt index fd8205c8ad..f1f039ca2e 100644 --- a/plugins/flytekit-deck-standard/requirements.txt +++ b/plugins/flytekit-deck-standard/requirements.txt @@ -6,51 +6,99 @@ # -e file:.#egg=flytekitplugins-deck-standard # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time -attrs==22.2.0 - # via visions + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via + # aiohttp + # visions +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs backcall==0.2.0 # via ipython binaryornot==0.4.4 # via cookiecutter -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage cycler==0.11.0 # via matplotlib -dataclasses-json==0.5.7 +dacite==1.8.1 + # via ydata-profiling +dataclasses-json==0.5.9 # via flytekit -debugpy==1.6.6 +debugpy==1.6.7 # via ipykernel decorator==5.1.1 # via + # gcsfs # ipython - # retry -deprecated==1.2.13 +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit @@ -58,15 +106,57 @@ docstring-parser==0.15 # via flytekit entrypoints==0.4 # via jupyter-client -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-deck-standard +flytekit==1.2.14 # via flytekitplugins-deck-standard fonttools==4.38.0 # via matplotlib -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -77,15 +167,21 @@ grpcio-status==1.48.2 htmlmin==0.1.12 # via ydata-profiling idna==3.4 - # via requests + # via + # requests + # yarl imagehash==4.3.1 - # via visions -importlib-metadata==6.0.0 # via + # visions + # ydata-profiling +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit # keyring # markdown + # rich-click importlib-resources==5.12.0 # via keyring ipykernel==6.16.2 @@ -94,8 +190,10 @@ ipython==7.34.0 # via # ipykernel # ipywidgets -ipywidgets==8.0.4 +ipywidgets==8.0.7 # via flytekitplugins-deck-standard +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jedi==0.18.2 @@ -107,11 +205,10 @@ jeepney==0.8.0 jinja2==3.1.2 # via # cookiecutter - # jinja2-time # ydata-profiling -jinja2-time==0.2.0 - # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 # via # flytekit # phik @@ -119,15 +216,19 @@ jupyter-client==7.4.9 # via ipykernel jupyter-core==4.12.0 # via jupyter-client -jupyterlab-widgets==3.0.5 +jupyterlab-widgets==3.0.8 # via ipywidgets -keyring==23.13.1 +keyring==24.1.1 # via flytekit kiwisolver==1.4.4 # via matplotlib -markdown==3.4.1 +kubernetes==26.1.0 + # via flytekit +markdown==3.4.3 # via flytekitplugins-deck-standard -markupsafe==2.1.2 +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -142,20 +243,34 @@ matplotlib==3.5.3 # via # phik # seaborn + # wordcloud # ydata-profiling matplotlib-inline==0.1.6 # via # ipykernel # ipython -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl multimethod==1.9.1 # via # visions # ydata-profiling mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit nest-asyncio==1.5.6 # via @@ -172,17 +287,22 @@ numpy==1.21.6 # patsy # phik # pyarrow + # pywavelets # scipy # seaborn # statsmodels # visions + # wordcloud # ydata-profiling -packaging==23.0 +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 # via # docker # ipykernel # marshmallow # matplotlib + # plotly # statsmodels pandas==1.3.5 # via @@ -204,57 +324,71 @@ phik==0.12.3 # via ydata-profiling pickleshare==0.7.5 # via ipython -pillow==9.4.0 +pillow==9.5.0 # via # imagehash # matplotlib # visions -plotly==5.13.0 + # wordcloud +plotly==5.15.0 # via flytekitplugins-deck-standard -prompt-toolkit==3.0.37 +portalocker==2.7.0 + # via msal-extensions +prompt-toolkit==3.0.39 # via ipython protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -psutil==5.9.4 +psutil==5.9.5 # via ipykernel ptyprocess==0.7.0 # via pexpect -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pydantic==1.10.5 +pydantic==1.10.11 # via ydata-profiling -pygments==2.14.0 - # via ipython -pyopenssl==23.0.0 +pygments==2.15.1 + # via + # ipython + # rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit -pyparsing==3.0.9 +pyparsing==3.1.0 # via matplotlib python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit # jupyter-client + # kubernetes # matplotlib # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -264,23 +398,45 @@ pyyaml==6.0 # via # cookiecutter # flytekit + # kubernetes + # responses # ydata-profiling -pyzmq==25.0.0 +pyzmq==25.1.0 # via # ipykernel # jupyter-client -regex==2022.10.31 +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib # responses # ydata-profiling -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 + # via flytekit +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 # via flytekit -retry==0.9.2 +rsa==4.9 + # via google-auth +s3fs==2023.1.0 # via flytekit scipy==1.7.3 # via @@ -296,9 +452,16 @@ singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity + # google-auth # grpcio + # isodate + # kubernetes # patsy # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 @@ -307,17 +470,15 @@ statsmodels==0.13.5 # via ydata-profiling tangled-up-in-unicode==0.2.0 # via visions -tenacity==8.2.1 +tenacity==8.2.2 # via plotly text-unidecode==1.3 # via python-slugify -toml==0.10.2 - # via responses tornado==6.2 # via # ipykernel # jupyter-client -tqdm==4.64.1 +tqdm==4.65.0 # via ydata-profiling traitlets==5.9.0 # via @@ -329,43 +490,63 @@ traitlets==5.9.0 # matplotlib-inline typeguard==2.13.3 # via ydata-profiling -types-toml==0.10.8.5 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # importlib-metadata # kiwisolver + # markdown-it-py # pydantic + # pyjwt # responses + # rich # seaborn # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses visions[type_image_path]==0.7.5 # via ydata-profiling wcwidth==0.2.6 # via prompt-toolkit -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via flytekit -widgetsnbextension==4.0.5 +widgetsnbextension==4.0.8 # via ipywidgets -wrapt==1.14.1 +wordcloud==1.9.2 + # via ydata-profiling +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -ydata-profiling==4.0.0 +yarl==1.9.2 + # via aiohttp +ydata-profiling==4.3.1 # via pandas-profiling -zipp==3.14.0 +zipp==3.15.0 # via # importlib-metadata # importlib-resources diff --git a/plugins/flytekit-deck-standard/setup.py b/plugins/flytekit-deck-standard/setup.py index 7f2d14ae12..36edb211d1 100644 --- a/plugins/flytekit-deck-standard/setup.py +++ b/plugins/flytekit-deck-standard/setup.py @@ -4,7 +4,14 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}-standard" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "markdown", "plotly", "pandas_profiling", "ipywidgets"] +plugin_requires = [ + "flytekit>=1.1.0b0,<1.3.0,<2.0.0", + "markdown", + "plotly", + "pandas_profiling", + "ipywidgets", + "flyteidl>=1.2.11,<1.3.0", +] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt index e5f0595edc..28f176957f 100644 --- a/plugins/flytekit-dolt/requirements.txt +++ b/plugins/flytekit-dolt/requirements.txt @@ -6,43 +6,89 @@ # -e file:.#egg=flytekitplugins-dolt # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via # dolt-integrations # flytekit decorator==5.1.1 - # via retry -deprecated==1.2.13 + # via gcsfs +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit @@ -52,13 +98,55 @@ dolt-integrations==0.1.5 # via flytekitplugins-dolt doltcli==0.1.18 # via dolt-integrations -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-dolt +flytekit==1.2.14 # via flytekitplugins-dolt -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -67,14 +155,20 @@ grpcio==1.48.2 grpcio-status==1.48.2 # via flytekit idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit # keyring + # rich-click importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jeepney==0.8.0 @@ -82,16 +176,18 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 + # via flytekit +keyring==24.1.1 # via flytekit -keyring==23.13.1 +kubernetes==26.1.0 # via flytekit -markupsafe==2.1.2 +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -102,18 +198,33 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit numpy==1.21.6 # via # flytekit # pandas # pyarrow -packaging==23.0 +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 # via # docker # marshmallow @@ -121,36 +232,49 @@ pandas==1.3.5 # via # dolt-integrations # flytekit +portalocker==2.7.0 + # via msal-extensions protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -158,17 +282,39 @@ pyyaml==6.0 # via # cookiecutter # flytekit -regex==2022.10.31 + # kubernetes + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib # responses -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.1.0 # via flytekit secretstorage==3.3.3 # via keyring @@ -176,44 +322,70 @@ singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -toml==0.10.2 - # via responses -typed-ast==1.5.4 +typed-ast==1.5.5 # via doltcli -types-toml==0.10.8.5 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # importlib-metadata + # markdown-it-py + # pyjwt # responses + # rich # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/plugins/flytekit-dolt/setup.py b/plugins/flytekit-dolt/setup.py index fb351222e4..7171bf692d 100644 --- a/plugins/flytekit-dolt/setup.py +++ b/plugins/flytekit-dolt/setup.py @@ -6,7 +6,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "dolt_integrations>=0.1.5"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "dolt_integrations>=0.1.5", "flyteidl>=1.2.11,<1.3.0"] dev_requires = ["pytest-mock>=3.6.1"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-duckdb/requirements.txt b/plugins/flytekit-duckdb/requirements.txt index 6996002a15..6d74dfc049 100644 --- a/plugins/flytekit-duckdb/requirements.txt +++ b/plugins/flytekit-duckdb/requirements.txt @@ -1,68 +1,150 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.7 # by the following command: # # pip-compile requirements.in # -e file:.#egg=flytekitplugins-duckdb # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -cachetools==5.3.0 +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 # via google-auth -certifi==2022.12.7 +certifi==2023.5.7 # via # kubernetes # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit -cloudpickle==2.2.0 + # rich-click +cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.0 - # via pyopenssl -dataclasses-json==0.5.7 +cryptography==41.0.1 + # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt + # pyopenssl + # secretstorage +dataclasses-json==0.5.9 # via flytekit -deprecated==1.2.13 +decorator==5.1.1 + # via gcsfs +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -duckdb==0.6.1 +duckdb==0.8.1 # via # -r requirements.in # flytekitplugins-duckdb -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.9 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-duckdb +flytekit==1.2.14 # via flytekitplugins-duckdb +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit gitdb==4.0.10 # via gitpython -gitpython==3.1.31 +gitpython==3.1.32 # via flytekit -google-auth==2.16.2 - # via kubernetes -googleapis-common-protos==1.58.0 +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -71,30 +153,39 @@ grpcio==1.48.2 grpcio-status==1.48.2 # via flytekit idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit - # jsonschema # keyring -importlib-resources==5.10.2 + # rich-click +importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring -jinja2==3.1.2 +jeepney==0.8.0 # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 + # keyring + # secretstorage +jinja2==3.1.2 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 # via flytekit -keyring==23.13.1 +keyring==24.1.1 # via flytekit kubernetes==26.1.0 # via flytekit -markupsafe==2.1.1 +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -105,30 +196,45 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes -mypy-extensions==0.4.3 +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl +mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit -numpy==1.23.5 +numpy==1.21.6 # via - # duckdb # flytekit # pandas # pyarrow oauthlib==3.2.2 # via requests-oauthlib -packaging==23.0 +packaging==23.1 # via # docker # marshmallow -pandas==1.5.2 +pandas==1.3.5 # via flytekit +portalocker==2.7.0 + # via msal-extensions protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger @@ -136,30 +242,35 @@ protoc-gen-swagger==0.1.0 # via flyteidl pyarrow==10.0.1 # via flytekit -pyasn1==0.4.8 +pyasn1==0.5.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 +pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit # kubernetes # pandas -python-json-logger==2.0.4 +python-json-logger==2.0.7 # via flytekit -python-slugify==7.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7 +pytz==2023.3 # via # flytekit # pandas @@ -168,28 +279,50 @@ pyyaml==6.0 # cookiecutter # flytekit # kubernetes -regex==2022.10.31 + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage # kubernetes + # msal # requests-oauthlib # responses requests-oauthlib==1.3.1 - # via kubernetes -responses==0.22.0 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 # via flytekit rsa==4.9 # via google-auth +s3fs==2023.1.0 + # via flytekit +secretstorage==3.3.3 + # via keyring +singledispatchmethod==1.0 + # via flytekit six==1.16.0 # via + # azure-core + # azure-identity # google-auth # grpcio + # isodate # kubernetes # python-dateutil smmap==5.0.0 @@ -200,39 +333,53 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -toml==0.10.2 - # via responses -types-toml==0.10.8.1 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.4.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit # gitpython # importlib-metadata + # markdown-it-py + # pyjwt # responses + # rich # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth # kubernetes # requests # responses -websocket-client==1.4.2 +websocket-client==1.6.1 # via # docker # kubernetes -wheel==0.38.4 +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.11.0 - # via importlib-metadata +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/plugins/flytekit-duckdb/setup.py b/plugins/flytekit-duckdb/setup.py index 62a0d53845..091188e652 100644 --- a/plugins/flytekit-duckdb/setup.py +++ b/plugins/flytekit-duckdb/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit<1.3.0,<2.0.0", "duckdb"] +plugin_requires = ["flytekit<1.3.0,<2.0.0", "duckdb", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py index fec6647443..2e107b7848 100644 --- a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py +++ b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py @@ -16,7 +16,7 @@ class EnvdImageSpecBuilder(ImageSpecBuilder): def build_image(self, image_spec: ImageSpec): cfg_path = create_envd_config(image_spec) - command = f"envd build --path {pathlib.Path(cfg_path).parent}" + command = f"envd build --path {pathlib.Path(cfg_path).parent} --platform {image_spec.platform}" if image_spec.registry: command += f" --output type=image,name={image_spec.image_name()},push=true" click.secho(f"Run command: {command} ", fg="blue") diff --git a/plugins/flytekit-envd/requirements.txt b/plugins/flytekit-envd/requirements.txt index c04adff810..99340ac386 100644 --- a/plugins/flytekit-envd/requirements.txt +++ b/plugins/flytekit-envd/requirements.txt @@ -1,14 +1,14 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.7 # by the following command: # # pip-compile requirements.in # -e file:.#egg=flytekitplugins-envd # via -r requirements.in -adlfs==2023.4.0 +adlfs==2022.2.0 # via flytekit -aiobotocore==2.5.0 +aiobotocore==2.4.2 # via s3fs aiohttp==3.8.4 # via @@ -21,12 +21,14 @@ aioitertools==0.11.0 aiosignal==1.3.1 # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter async-timeout==4.0.2 # via aiohttp +asynctest==0.13.0 + # via aiohttp attrs==23.1.0 # via aiohttp -azure-core==1.26.4 +azure-core==1.28.0 # via # adlfs # azure-identity @@ -39,11 +41,11 @@ azure-storage-blob==12.16.0 # via adlfs binaryornot==0.4.4 # via cookiecutter -botocore==1.29.76 +botocore==1.27.59 # via aiobotocore -cachetools==5.3.0 +cachetools==5.3.1 # via google-auth -certifi==2022.12.7 +certifi==2023.5.7 # via # kubernetes # requests @@ -53,69 +55,72 @@ cffi==1.15.1 # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # aiohttp # requests -click==8.1.3 +click==8.1.4 # via # cookiecutter # flytekit # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==40.0.1 +cryptography==41.0.1 # via # azure-identity # azure-storage-blob # msal # pyjwt # pyopenssl -dataclasses-json==0.5.7 + # secretstorage +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 # via gcsfs -deprecated==1.2.13 +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -envd==0.3.16 +envd==0.3.33 # via flytekitplugins-envd -flyteidl==1.2.10 - # via flytekit -flytekit==1.2.12 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-envd +flytekit==1.2.14 # via flytekitplugins-envd frozenlist==1.3.3 # via # aiohttp # aiosignal -fsspec==2023.5.0 +fsspec==2023.1.0 # via # adlfs # flytekit # gcsfs # s3fs -gcsfs==2023.5.0 +gcsfs==2023.1.0 # via flytekit gitdb==4.0.10 # via gitpython -gitpython==3.1.31 +gitpython==3.1.32 # via flytekit -google-api-core==2.11.0 +google-api-core==2.11.1 # via # google-cloud-core # google-cloud-storage -google-auth==2.17.1 +google-auth==2.21.0 # via # gcsfs # google-api-core @@ -125,15 +130,15 @@ google-auth==2.17.1 # kubernetes google-auth-oauthlib==1.0.0 # via gcsfs -google-cloud-core==2.3.2 +google-cloud-core==2.3.3 # via google-cloud-storage -google-cloud-storage==2.9.0 +google-cloud-storage==2.10.0 # via gcsfs google-crc32c==1.5.0 # via google-resumable-media google-resumable-media==2.5.0 # via google-cloud-storage -googleapis-common-protos==1.59.0 +googleapis-common-protos==1.59.1 # via # flyteidl # flytekit @@ -149,31 +154,36 @@ idna==3.4 # via # requests # yarl -importlib-metadata==6.1.0 +importlib-metadata==6.7.0 # via + # attrs + # click # flytekit # keyring + # rich-click +importlib-resources==5.12.0 + # via keyring isodate==0.6.1 # via azure-storage-blob jaraco-classes==3.2.3 # via keyring -jinja2==3.1.2 +jeepney==0.8.0 # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 + # keyring + # secretstorage +jinja2==3.1.2 # via cookiecutter jmespath==1.0.1 # via botocore -joblib==1.2.0 +joblib==1.3.1 # via flytekit -keyring==23.13.1 +keyring==24.1.1 # via flytekit kubernetes==26.1.0 # via flytekit markdown-it-py==2.2.0 # via rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -201,20 +211,20 @@ multidict==6.0.4 # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.3.1 +natsort==8.4.0 # via flytekit -numpy==1.23.5 +numpy==1.21.6 # via # flytekit # pandas # pyarrow oauthlib==3.2.2 # via requests-oauthlib -packaging==23.0 +packaging==23.1 # via # docker # marshmallow -pandas==1.5.3 +pandas==1.3.5 # via flytekit portalocker==2.7.0 # via msal-extensions @@ -230,11 +240,11 @@ protoc-gen-swagger==0.1.0 # via flyteidl pyarrow==10.0.1 # via flytekit -pyasn1==0.4.8 +pyasn1==0.5.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 +pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi @@ -242,7 +252,7 @@ pygments==2.15.1 # via rich pyjwt[crypto]==2.7.0 # via msal -pyopenssl==23.1.1 +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via @@ -268,9 +278,9 @@ pyyaml==6.0 # flytekit # kubernetes # responses -regex==2023.3.23 +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via # azure-core # azure-datalake-store @@ -290,7 +300,7 @@ requests-oauthlib==1.3.1 # kubernetes responses==0.23.1 # via flytekit -rich==13.3.5 +rich==13.4.2 # via # flytekit # rich-click @@ -298,7 +308,11 @@ rich-click==1.6.1 # via flytekit rsa==4.9 # via google-auth -s3fs==2023.5.0 +s3fs==2023.1.0 + # via flytekit +secretstorage==3.3.3 + # via keyring +singledispatchmethod==1.0 # via flytekit six==1.16.0 # via @@ -317,26 +331,37 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -types-pyyaml==6.0.12.9 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp # aioitertools + # arrow + # async-timeout # azure-core # azure-storage-blob # flytekit + # gitpython + # importlib-metadata + # markdown-it-py + # pyjwt + # responses + # rich # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.15 +urllib3==1.26.16 # via # botocore # docker # flytekit + # google-auth # kubernetes # requests # responses -websocket-client==1.5.1 +websocket-client==1.6.1 # via # docker # kubernetes @@ -350,7 +375,9 @@ wrapt==1.15.0 yarl==1.9.2 # via aiohttp zipp==3.15.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/plugins/flytekit-envd/setup.py b/plugins/flytekit-envd/setup.py index 95801d721a..1ede731447 100644 --- a/plugins/flytekit-envd/setup.py +++ b/plugins/flytekit-envd/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit<1.3.0", "envd"] +plugin_requires = ["flytekit<1.3.0", "envd>=0.3.22", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-envd/tests/test_image_spec.py b/plugins/flytekit-envd/tests/test_image_spec.py index 7c7ccd2151..b4b5f1d245 100644 --- a/plugins/flytekit-envd/tests/test_image_spec.py +++ b/plugins/flytekit-envd/tests/test_image_spec.py @@ -16,6 +16,7 @@ def test_image_spec(): EnvdImageSpecBuilder().build_image(image_spec) config_path = create_envd_config(image_spec) + assert image_spec.platform == "linux/amd64" contents = Path(config_path).read_text() assert ( contents @@ -25,7 +26,7 @@ def build(): base(image="cr.flyte.org/flyteorg/flytekit:py3.8-latest", dev=False) install.python_packages(name = ["pandas"]) install.apt_packages(name = ["git"]) - runtime.environ(env={'PYTHONPATH': '/root', '_F_IMG_ID': 'flytekit:yZ8jICcDTLoDArmNHbWNwg..'}) + runtime.environ(env={'PYTHONPATH': '/root', '_F_IMG_ID': 'flytekit:OLFSrRjcG5_uXuRqd0TSdQ..'}) install.python(version="3.8") """ ) diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt index 1e4051a41c..15cd97fde6 100644 --- a/plugins/flytekit-greatexpectations/requirements.txt +++ b/plugins/flytekit-greatexpectations/requirements.txt @@ -6,9 +6,23 @@ # -e file:.#egg=flytekitplugins-great_expectations # via -r requirements.in -altair==4.2.0 +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp +altair==4.2.2 # via great-expectations -anyio==3.6.2 +anyio==3.7.1 # via jupyter-server argon2-cffi==21.3.0 # via @@ -18,64 +32,93 @@ argon2-cffi==21.3.0 argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.2.3 - # via jinja2-time -attrs==22.2.0 - # via jsonschema + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via + # aiohttp + # jsonschema +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs backcall==0.2.0 # via ipython backports-zoneinfo==0.2.1 - # via - # pytz-deprecation-shim - # tzlocal -beautifulsoup4==4.11.2 + # via tzlocal +beautifulsoup4==4.12.2 # via nbconvert binaryornot==0.4.4 # via cookiecutter bleach==6.0.0 # via nbconvert -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 # via # argon2-cffi-bindings + # azure-datalake-store # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit # great-expectations + # rich-click cloudpickle==2.2.1 # via flytekit colorama==0.4.6 # via great-expectations -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob # great-expectations + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit -debugpy==1.6.6 +debugpy==1.6.7 # via ipykernel decorator==5.1.1 # via + # gcsfs # ipython - # retry defusedxml==0.7.1 # via nbconvert -deprecated==1.2.13 +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit @@ -85,20 +128,62 @@ entrypoints==0.4 # via # altair # jupyter-client -fastjsonschema==2.16.2 +exceptiongroup==1.1.2 + # via anyio +fastjsonschema==2.17.1 # via nbformat -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-great-expectations +flytekit==1.2.14 # via flytekitplugins-great-expectations -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status -great-expectations==0.15.12 - # via - # -r requirements.in - # flytekitplugins-great-expectations +great-expectations==0.16.13 + # via flytekitplugins-great-expectations greenlet==2.0.2 # via sqlalchemy grpcio==1.48.2 @@ -111,8 +196,10 @@ idna==3.4 # via # anyio # requests -importlib-metadata==6.0.0 + # yarl +importlib-metadata==6.7.0 # via + # attrs # click # flytekit # great-expectations @@ -120,6 +207,7 @@ importlib-metadata==6.0.0 # keyring # nbconvert # nbformat + # rich-click # sqlalchemy importlib-resources==5.12.0 # via @@ -127,16 +215,22 @@ importlib-resources==5.12.0 # keyring ipykernel==6.16.2 # via + # ipywidgets # nbclassic # notebook ipython==7.34.0 # via # great-expectations # ipykernel + # ipywidgets ipython-genutils==0.2.0 # via # nbclassic # notebook +ipywidgets==8.0.7 + # via great-expectations +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jedi==0.18.2 @@ -150,18 +244,17 @@ jinja2==3.1.2 # altair # cookiecutter # great-expectations - # jinja2-time # jupyter-server # nbclassic # nbconvert # notebook -jinja2-time==0.2.0 - # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 # via flytekit -jsonpatch==1.32 +jsonpatch==1.33 # via great-expectations -jsonpointer==2.3 +jsonpointer==2.4 # via jsonpatch jsonschema==4.17.3 # via @@ -184,21 +277,30 @@ jupyter-core==4.12.0 # nbconvert # nbformat # notebook -jupyter-server==1.23.6 +jupyter-server==1.24.0 # via # nbclassic # notebook-shim jupyterlab-pygments==0.2.2 # via nbconvert -keyring==23.13.1 +jupyterlab-widgets==3.0.8 + # via ipywidgets +keyring==24.1.1 # via flytekit -markupsafe==2.1.2 +kubernetes==26.1.0 + # via flytekit +makefun==1.15.1 + # via great-expectations +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via # jinja2 # nbconvert marshmallow==3.19.0 # via # dataclasses-json + # great-expectations # marshmallow-enum # marshmallow-jsonschema marshmallow-enum==1.5.1 @@ -209,26 +311,39 @@ matplotlib-inline==0.1.6 # via # ipykernel # ipython -mistune==2.0.5 +mdurl==0.1.2 + # via markdown-it-py +mistune==3.0.1 # via # great-expectations # nbconvert -more-itertools==9.0.0 +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit -nbclassic==0.5.2 +nbclassic==1.0.0 # via notebook -nbclient==0.7.2 +nbclient==0.7.4 # via nbconvert -nbconvert==7.2.9 +nbconvert==7.6.0 # via # jupyter-server # nbclassic # notebook -nbformat==5.7.3 +nbformat==5.8.0 # via # great-expectations # jupyter-server @@ -242,9 +357,9 @@ nest-asyncio==1.5.6 # jupyter-client # nbclassic # notebook -notebook==6.5.2 +notebook==6.5.4 # via great-expectations -notebook-shim==0.2.2 +notebook-shim==0.2.3 # via nbclassic numpy==1.21.6 # via @@ -254,7 +369,9 @@ numpy==1.21.6 # pandas # pyarrow # scipy -packaging==23.0 +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 # via # docker # great-expectations @@ -277,98 +394,132 @@ pickleshare==0.7.5 # via ipython pkgutil-resolve-name==1.3.10 # via jsonschema -prometheus-client==0.16.0 +portalocker==2.7.0 + # via msal-extensions +prometheus-client==0.17.1 # via # jupyter-server # nbclassic # notebook -prompt-toolkit==3.0.37 +prompt-toolkit==3.0.39 # via ipython protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -psutil==5.9.4 +psutil==5.9.5 # via ipykernel ptyprocess==0.7.0 # via # pexpect # terminado -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pygments==2.14.0 +pydantic==1.10.11 + # via great-expectations +pygments==2.15.1 # via # ipython # nbconvert -pyopenssl==23.0.0 + # rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit -pyparsing==2.4.7 +pyparsing==3.1.0 # via great-expectations pyrsistent==0.19.3 # via jsonschema python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit # great-expectations # jupyter-client + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # great-expectations # pandas -pytz-deprecation-shim==0.1.0.post0 - # via tzlocal pyyaml==6.0 # via # cookiecutter # flytekit -pyzmq==25.0.0 + # kubernetes + # responses +pyzmq==25.1.0 # via # ipykernel # jupyter-client # jupyter-server # nbclassic # notebook -regex==2022.10.31 +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage # great-expectations + # kubernetes + # msal + # requests-oauthlib # responses -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 # via flytekit +rsa==4.9 + # via google-auth ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.7 # via ruamel-yaml +s3fs==2023.1.0 + # via flytekit scipy==1.7.3 # via great-expectations secretstorage==3.3.3 # via keyring -send2trash==1.8.0 +send2trash==1.8.2 # via # jupyter-server # nbclassic @@ -377,23 +528,28 @@ singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity # bleach + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sniffio==1.3.0 # via anyio sortedcontainers==2.4.0 # via flytekit -soupsieve==2.4 +soupsieve==2.4.1 # via beautifulsoup4 -sqlalchemy==1.4.39 +sqlalchemy==2.0.18 # via # -r requirements.in # flytekitplugins-great-expectations statsd==3.3.0 # via flytekit -termcolor==2.2.0 - # via great-expectations terminado==0.17.1 # via # jupyter-server @@ -403,8 +559,6 @@ text-unidecode==1.3 # via python-slugify tinycss2==1.2.1 # via nbconvert -toml==0.10.2 - # via responses toolz==0.12.0 # via altair tornado==6.2 @@ -415,12 +569,13 @@ tornado==6.2 # nbclassic # notebook # terminado -tqdm==4.64.1 +tqdm==4.65.0 # via great-expectations traitlets==5.9.0 # via # ipykernel # ipython + # ipywidgets # jupyter-client # jupyter-core # jupyter-server @@ -430,30 +585,43 @@ traitlets==5.9.0 # nbconvert # nbformat # notebook -types-toml==0.10.8.5 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # anyio # argon2-cffi # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # great-expectations # importlib-metadata # jsonschema + # markdown-it-py + # pydantic + # pyjwt # responses + # rich + # sqlalchemy # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -tzdata==2022.7 - # via pytz-deprecation-shim -tzlocal==4.2 +tzlocal==5.0.1 # via great-expectations -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth # great-expectations + # kubernetes # requests # responses wcwidth==0.2.6 @@ -462,17 +630,23 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.5.1 +websocket-client==1.6.1 # via # docker # jupyter-server -wheel==0.38.4 + # kubernetes +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +widgetsnbextension==4.0.8 + # via ipywidgets +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources diff --git a/plugins/flytekit-greatexpectations/setup.py b/plugins/flytekit-greatexpectations/setup.py index ac646f3208..e377ec3e94 100644 --- a/plugins/flytekit-greatexpectations/setup.py +++ b/plugins/flytekit-greatexpectations/setup.py @@ -4,7 +4,12 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "great-expectations>=0.13.30", "sqlalchemy>=1.4.23"] +plugin_requires = [ + "flytekit>=1.1.0b0,<1.3.0,<2.0.0", + "great-expectations>=0.13.30", + "sqlalchemy>=1.4.23", + "flyteidl>=1.2.11,<1.3.0", +] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt index 50dfef8807..a90e5defaf 100644 --- a/plugins/flytekit-hive/requirements.txt +++ b/plugins/flytekit-hive/requirements.txt @@ -6,53 +6,141 @@ # -e file:.#egg=flytekitplugins-hive # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 - # via retry -deprecated==1.2.13 + # via gcsfs +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-hive +flytekit==1.2.14 # via flytekitplugins-hive -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -61,14 +149,20 @@ grpcio==1.48.2 grpcio-status==1.48.2 # via flytekit idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit # keyring + # rich-click importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jeepney==0.8.0 @@ -76,16 +170,18 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 + # via flytekit +keyring==24.1.1 # via flytekit -keyring==23.13.1 +kubernetes==26.1.0 # via flytekit -markupsafe==2.1.2 +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -96,53 +192,81 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit numpy==1.21.6 # via # flytekit # pandas # pyarrow -packaging==23.0 +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 # via # docker # marshmallow pandas==1.3.5 # via flytekit +portalocker==2.7.0 + # via msal-extensions protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -150,17 +274,39 @@ pyyaml==6.0 # via # cookiecutter # flytekit -regex==2022.10.31 + # kubernetes + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib # responses -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.1.0 # via flytekit secretstorage==3.3.3 # via keyring @@ -168,42 +314,68 @@ singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -toml==0.10.2 - # via responses -types-toml==0.10.8.5 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # importlib-metadata + # markdown-it-py + # pyjwt # responses + # rich # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/plugins/flytekit-hive/setup.py b/plugins/flytekit-hive/setup.py index 80191042bd..192f70ed1d 100644 --- a/plugins/flytekit-hive/setup.py +++ b/plugins/flytekit-hive/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-huggingface/requirements.txt b/plugins/flytekit-huggingface/requirements.txt index 07cefbae2c..c2821305e2 100644 --- a/plugins/flytekit-huggingface/requirements.txt +++ b/plugins/flytekit-huggingface/requirements.txt @@ -6,81 +6,153 @@ # -e file:.#egg=flytekitplugins-huggingface # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs aiohttp==3.8.4 # via + # adlfs + # aiobotocore # datasets # fsspec + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore aiosignal==1.3.1 # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter async-timeout==4.0.2 # via aiohttp asynctest==0.13.0 # via aiohttp -attrs==22.2.0 +attrs==23.1.0 # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 +charset-normalizer==3.2.0 # via # aiohttp # requests -click==8.1.3 +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit -datasets==2.10.0 +datasets==2.13.1 # via flytekitplugins-huggingface decorator==5.1.1 - # via retry -deprecated==1.2.13 + # via gcsfs +deprecated==1.2.14 # via flytekit dill==0.3.6 # via # datasets # multiprocess -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -filelock==3.9.0 +filelock==3.12.2 # via huggingface-hub -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-huggingface +flytekit==1.2.14 # via flytekitplugins-huggingface frozenlist==1.3.3 # via # aiohttp # aiosignal fsspec[http]==2023.1.0 - # via datasets -googleapis-common-protos==1.58.0 + # via + # adlfs + # datasets + # flytekit + # gcsfs + # huggingface-hub + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -88,21 +160,25 @@ grpcio==1.48.2 # grpcio-status grpcio-status==1.48.2 # via flytekit -huggingface-hub==0.12.1 +huggingface-hub==0.16.4 # via datasets idna==3.4 # via # requests # yarl -importlib-metadata==6.0.0 +importlib-metadata==6.7.0 # via + # attrs # click # datasets # flytekit # huggingface-hub # keyring + # rich-click importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jeepney==0.8.0 @@ -110,16 +186,18 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 + # via flytekit +keyring==24.1.1 # via flytekit -keyring==23.13.1 +kubernetes==26.1.0 # via flytekit -markupsafe==2.1.2 +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -130,8 +208,17 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity multidict==6.0.4 # via # aiohttp @@ -140,7 +227,7 @@ multiprocess==0.70.14 # via datasets mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit numpy==1.21.6 # via @@ -148,7 +235,9 @@ numpy==1.21.6 # flytekit # pandas # pyarrow -packaging==23.0 +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 # via # datasets # docker @@ -158,38 +247,51 @@ pandas==1.3.5 # via # datasets # flytekit +portalocker==2.7.0 + # via msal-extensions protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==10.0.1 # via # datasets # flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -199,22 +301,42 @@ pyyaml==6.0 # datasets # flytekit # huggingface-hub -regex==2022.10.31 + # kubernetes + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # datasets # docker # flytekit # fsspec + # gcsfs + # google-api-core + # google-cloud-storage # huggingface-hub + # kubernetes + # msal + # requests-oauthlib # responses -responses==0.18.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 + # via flytekit +rich==13.4.2 # via - # datasets # flytekit -retry==0.9.2 + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.1.0 # via flytekit secretstorage==3.3.3 # via keyring @@ -222,49 +344,75 @@ singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -tqdm==4.64.1 +tqdm==4.65.0 # via # datasets # huggingface-hub -typing-extensions==4.5.0 +types-pyyaml==6.0.12.10 + # via responses +typing-extensions==4.7.1 # via # aiohttp + # aioitertools # arrow # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # huggingface-hub # importlib-metadata + # markdown-it-py + # pyjwt + # responses + # rich # typing-inspect # yarl -typing-inspect==0.8.0 +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit xxhash==3.2.0 # via datasets -yarl==1.8.2 +yarl==1.9.2 # via aiohttp -zipp==3.14.0 +zipp==3.15.0 # via # importlib-metadata # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/plugins/flytekit-huggingface/setup.py b/plugins/flytekit-huggingface/setup.py index acdbc20810..78a741d7bd 100644 --- a/plugins/flytekit-huggingface/setup.py +++ b/plugins/flytekit-huggingface/setup.py @@ -7,6 +7,7 @@ plugin_requires = [ "flytekit>=1.1.0b0,<1.3.0,<2.0.0", "datasets>=2.4.0", + "flyteidl>=1.2.11,<1.3.0", ] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt index 9c911b1263..8df677f7e1 100644 --- a/plugins/flytekit-k8s-pod/requirements.txt +++ b/plugins/flytekit-k8s-pod/requirements.txt @@ -6,59 +6,141 @@ # -e file:.#egg=flytekitplugins-pod # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -cachetools==5.3.0 +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 # via google-auth -certifi==2022.12.7 +certifi==2023.5.7 # via # kubernetes # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 - # via retry -deprecated==1.2.13 + # via gcsfs +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-pod +flytekit==1.2.14 # via flytekitplugins-pod -google-auth==2.16.1 - # via kubernetes -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -67,14 +149,20 @@ grpcio==1.48.2 grpcio-status==1.48.2 # via flytekit idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit # keyring + # rich-click importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jeepney==0.8.0 @@ -82,18 +170,20 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 # via flytekit -keyring==23.13.1 +keyring==24.1.1 # via flytekit kubernetes==26.1.0 - # via flytekitplugins-pod -markupsafe==2.1.2 + # via + # flytekit + # flytekitplugins-pod +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -104,11 +194,24 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit numpy==1.21.6 # via @@ -117,49 +220,55 @@ numpy==1.21.6 # pyarrow oauthlib==3.2.2 # via requests-oauthlib -packaging==23.0 +packaging==23.1 # via # docker # marshmallow pandas==1.3.5 # via flytekit +portalocker==2.7.0 + # via msal-extensions protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit -pyasn1==0.4.8 +pyasn1==0.5.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 +pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -168,71 +277,104 @@ pyyaml==6.0 # cookiecutter # flytekit # kubernetes -regex==2022.10.31 + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage # kubernetes + # msal # requests-oauthlib # responses requests-oauthlib==1.3.1 - # via kubernetes -responses==0.22.0 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 # via flytekit rsa==4.9 # via google-auth +s3fs==2023.1.0 + # via flytekit secretstorage==3.3.3 # via keyring singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity # google-auth # grpcio + # isodate # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -toml==0.10.2 +types-pyyaml==6.0.12.10 # via responses -types-toml==0.10.8.5 - # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # importlib-metadata + # markdown-it-py + # pyjwt # responses + # rich # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth # kubernetes # requests # responses -websocket-client==1.5.1 +websocket-client==1.6.1 # via # docker # kubernetes -wheel==0.38.4 +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources diff --git a/plugins/flytekit-k8s-pod/setup.py b/plugins/flytekit-k8s-pod/setup.py index 3d9a20f124..df9c5c1fa9 100644 --- a/plugins/flytekit-k8s-pod/setup.py +++ b/plugins/flytekit-k8s-pod/setup.py @@ -7,6 +7,7 @@ plugin_requires = [ "flytekit>=1.1.0b0,<1.3.0,<2.0.0", "kubernetes>=12.0.1", + "flyteidl>=1.2.11,<1.3.0", ] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt index 1a9b0fcb54..d137df7e38 100644 --- a/plugins/flytekit-kf-mpi/requirements.txt +++ b/plugins/flytekit-kf-mpi/requirements.txt @@ -75,6 +75,7 @@ cryptography==41.0.1 # msal # pyjwt # pyopenssl + # secretstorage dataclasses-json==0.5.8 # via flytekit decorator==5.1.1 @@ -159,6 +160,10 @@ isodate==0.6.1 # via azure-storage-blob jaraco-classes==3.2.3 # via keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.2 # via # cookiecutter @@ -302,6 +307,8 @@ rsa==4.9 # via google-auth s3fs==2023.6.0 # via flytekit +secretstorage==3.3.3 + # via keyring six==1.16.0 # via # azure-core diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt index 3bdac2d459..2f17e07896 100644 --- a/plugins/flytekit-kf-pytorch/requirements.txt +++ b/plugins/flytekit-kf-pytorch/requirements.txt @@ -1,14 +1,14 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.7 # by the following command: # # pip-compile requirements.in # -e file:.#egg=flytekitplugins-kfpytorch # via -r requirements.in -adlfs==2023.4.0 +adlfs==2022.2.0 # via flytekit -aiobotocore==2.5.1 +aiobotocore==2.4.2 # via s3fs aiohttp==3.8.4 # via @@ -21,12 +21,14 @@ aioitertools==0.11.0 aiosignal==1.3.1 # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter async-timeout==4.0.2 # via aiohttp +asynctest==0.13.0 + # via aiohttp attrs==23.1.0 # via aiohttp -azure-core==1.27.1 +azure-core==1.28.0 # via # adlfs # azure-identity @@ -39,7 +41,7 @@ azure-storage-blob==12.16.0 # via adlfs binaryornot==0.4.4 # via cookiecutter -botocore==1.29.161 +botocore==1.27.59 # via aiobotocore cachetools==5.3.1 # via google-auth @@ -53,11 +55,11 @@ cffi==1.15.1 # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # aiohttp # requests -click==8.1.3 +click==8.1.4 # via # cookiecutter # flytekit @@ -66,7 +68,7 @@ cloudpickle==2.2.1 # via # flytekit # flytekitplugins-kfpytorch -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit croniter==1.4.1 # via flytekit @@ -77,7 +79,8 @@ cryptography==41.0.1 # msal # pyjwt # pyopenssl -dataclasses-json==0.5.8 + # secretstorage +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 # via gcsfs @@ -95,23 +98,23 @@ flyteidl==1.2.11 # via # flytekit # flytekitplugins-kfpytorch -flytekit==1.2.13 +flytekit==1.2.14 # via flytekitplugins-kfpytorch frozenlist==1.3.3 # via # aiohttp # aiosignal -fsspec==2023.6.0 +fsspec==2023.1.0 # via # adlfs # flytekit # gcsfs # s3fs -gcsfs==2023.6.0 +gcsfs==2023.1.0 # via flytekit gitdb==4.0.10 # via gitpython -gitpython==3.1.31 +gitpython==3.1.32 # via flytekit google-api-core==2.11.1 # via @@ -127,7 +130,7 @@ google-auth==2.21.0 # kubernetes google-auth-oauthlib==1.0.0 # via gcsfs -google-cloud-core==2.3.2 +google-cloud-core==2.3.3 # via google-cloud-storage google-cloud-storage==2.10.0 # via gcsfs @@ -153,29 +156,32 @@ idna==3.4 # yarl importlib-metadata==6.7.0 # via + # attrs + # click # flytekit # keyring + # rich-click importlib-resources==5.12.0 # via keyring isodate==0.6.1 # via azure-storage-blob jaraco-classes==3.2.3 # via keyring -jinja2==3.1.2 +jeepney==0.8.0 # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 + # keyring + # secretstorage +jinja2==3.1.2 # via cookiecutter jmespath==1.0.1 # via botocore joblib==1.3.1 # via flytekit -keyring==24.2.0 +keyring==24.1.1 # via flytekit kubernetes==26.1.0 # via flytekit -markdown-it-py==3.0.0 +markdown-it-py==2.2.0 # via rich markupsafe==2.1.3 # via jinja2 @@ -207,7 +213,7 @@ mypy-extensions==1.0.0 # via typing-inspect natsort==8.4.0 # via flytekit -numpy==1.23.5 +numpy==1.21.6 # via # flytekit # pandas @@ -218,7 +224,7 @@ packaging==23.1 # via # docker # marshmallow -pandas==1.5.3 +pandas==1.3.5 # via flytekit portalocker==2.7.0 # via msal-extensions @@ -302,7 +308,11 @@ rich-click==1.6.1 # via flytekit rsa==4.9 # via google-auth -s3fs==2023.6.0 +s3fs==2023.1.0 + # via flytekit +secretstorage==3.3.3 + # via keyring +singledispatchmethod==1.0 # via flytekit six==1.16.0 # via @@ -323,14 +333,23 @@ text-unidecode==1.3 # via python-slugify types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.7.0 +typing-extensions==4.7.1 # via + # aiohttp # aioitertools + # arrow + # async-timeout # azure-core # azure-storage-blob # flytekit + # gitpython + # importlib-metadata + # markdown-it-py + # pyjwt + # responses # rich # typing-inspect + # yarl typing-inspect==0.9.0 # via dataclasses-json urllib3==1.26.16 diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt index 70e3e1a7b2..c59cff36fa 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.txt +++ b/plugins/flytekit-kf-tensorflow/requirements.txt @@ -1,14 +1,14 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.7 # by the following command: # # pip-compile requirements.in # -e file:.#egg=flytekitplugins-kftensorflow # via -r requirements.in -adlfs==2023.4.0 +adlfs==2022.2.0 # via flytekit -aiobotocore==2.5.1 +aiobotocore==2.4.2 # via s3fs aiohttp==3.8.4 # via @@ -21,12 +21,14 @@ aioitertools==0.11.0 aiosignal==1.3.1 # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter async-timeout==4.0.2 # via aiohttp +asynctest==0.13.0 + # via aiohttp attrs==23.1.0 # via aiohttp -azure-core==1.27.1 +azure-core==1.28.0 # via # adlfs # azure-identity @@ -39,7 +41,7 @@ azure-storage-blob==12.16.0 # via adlfs binaryornot==0.4.4 # via cookiecutter -botocore==1.29.161 +botocore==1.27.59 # via aiobotocore cachetools==5.3.1 # via google-auth @@ -53,18 +55,18 @@ cffi==1.15.1 # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # aiohttp # requests -click==8.1.3 +click==8.1.4 # via # cookiecutter # flytekit # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit croniter==1.4.1 # via flytekit @@ -75,7 +77,8 @@ cryptography==41.0.1 # msal # pyjwt # pyopenssl -dataclasses-json==0.5.8 + # secretstorage +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 # via gcsfs @@ -93,23 +96,23 @@ flyteidl==1.2.11 # via # flytekit # flytekitplugins-kftensorflow -flytekit==1.2.13 +flytekit==1.2.14 # via flytekitplugins-kftensorflow frozenlist==1.3.3 # via # aiohttp # aiosignal -fsspec==2023.6.0 +fsspec==2023.1.0 # via # adlfs # flytekit # gcsfs # s3fs -gcsfs==2023.6.0 +gcsfs==2023.1.0 # via flytekit gitdb==4.0.10 # via gitpython -gitpython==3.1.31 +gitpython==3.1.32 # via flytekit google-api-core==2.11.1 # via @@ -125,7 +128,7 @@ google-auth==2.21.0 # kubernetes google-auth-oauthlib==1.0.0 # via gcsfs -google-cloud-core==2.3.2 +google-cloud-core==2.3.3 # via google-cloud-storage google-cloud-storage==2.10.0 # via gcsfs @@ -151,29 +154,32 @@ idna==3.4 # yarl importlib-metadata==6.7.0 # via + # attrs + # click # flytekit # keyring + # rich-click importlib-resources==5.12.0 # via keyring isodate==0.6.1 # via azure-storage-blob jaraco-classes==3.2.3 # via keyring -jinja2==3.1.2 +jeepney==0.8.0 # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 + # keyring + # secretstorage +jinja2==3.1.2 # via cookiecutter jmespath==1.0.1 # via botocore joblib==1.3.1 # via flytekit -keyring==24.2.0 +keyring==24.1.1 # via flytekit kubernetes==26.1.0 # via flytekit -markdown-it-py==3.0.0 +markdown-it-py==2.2.0 # via rich markupsafe==2.1.3 # via jinja2 @@ -205,7 +211,7 @@ mypy-extensions==1.0.0 # via typing-inspect natsort==8.4.0 # via flytekit -numpy==1.23.5 +numpy==1.21.6 # via # flytekit # pandas @@ -216,7 +222,7 @@ packaging==23.1 # via # docker # marshmallow -pandas==1.5.3 +pandas==1.3.5 # via flytekit portalocker==2.7.0 # via msal-extensions @@ -300,7 +306,11 @@ rich-click==1.6.1 # via flytekit rsa==4.9 # via google-auth -s3fs==2023.6.0 +s3fs==2023.1.0 + # via flytekit +secretstorage==3.3.3 + # via keyring +singledispatchmethod==1.0 # via flytekit six==1.16.0 # via @@ -321,14 +331,23 @@ text-unidecode==1.3 # via python-slugify types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.7.0 +typing-extensions==4.7.1 # via + # aiohttp # aioitertools + # arrow + # async-timeout # azure-core # azure-storage-blob # flytekit + # gitpython + # importlib-metadata + # markdown-it-py + # pyjwt + # responses # rich # typing-inspect + # yarl typing-inspect==0.9.0 # via dataclasses-json urllib3==1.26.16 diff --git a/plugins/flytekit-mlflow/requirements.txt b/plugins/flytekit-mlflow/requirements.txt index 03873c05f5..c4ec1da6ec 100644 --- a/plugins/flytekit-mlflow/requirements.txt +++ b/plugins/flytekit-mlflow/requirements.txt @@ -1,25 +1,64 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: # # pip-compile requirements.in # -e file:.#egg=flytekitplugins-mlflow # via -r requirements.in +adlfs==2023.4.0 + # via flytekit +aiobotocore==2.5.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp alembic==1.8.1 # via mlflow arrow==1.2.3 # via jinja2-time +async-timeout==4.0.2 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.17.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter +botocore==1.29.161 + # via aiobotocore +cachetools==5.3.1 + # via google-auth certifi==2022.9.24 - # via requests + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.0.0 # via binaryornot charset-normalizer==2.1.1 - # via requests + # via + # aiohttp + # requests click==8.1.3 # via # cookiecutter @@ -27,6 +66,7 @@ click==8.1.3 # flask # flytekit # mlflow + # rich-click cloudpickle==2.2.0 # via # flytekit @@ -36,13 +76,19 @@ cookiecutter==2.1.1 croniter==1.3.7 # via flytekit cryptography==38.0.3 - # via pyopenssl + # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt + # pyopenssl + # secretstorage databricks-cli==0.17.3 # via mlflow dataclasses-json==0.5.7 # via flytekit decorator==5.1.1 - # via retry + # via gcsfs deprecated==1.2.13 # via flytekit diskcache==5.4.0 @@ -61,21 +107,61 @@ flask==2.2.2 # via # mlflow # prometheus-flask-exporter -flyteidl==1.1.22 - # via flytekit -flytekit==1.2.3 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-mlflow +flytekit==1.2.14 # via flytekitplugins-mlflow +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.6.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.6.0 + # via flytekit gitdb==4.0.9 # via gitpython gitpython==3.1.29 - # via mlflow -googleapis-common-protos==1.56.4 + # via + # flytekit + # mlflow +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.22.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status greenlet==2.0.1 # via sqlalchemy -grpcio==1.50.0 +grpcio==1.48.2 # via # flytekit # grpcio-status @@ -86,17 +172,25 @@ grpcio-status==1.48.2 gunicorn==20.1.0 # via mlflow idna==3.4 - # via requests + # via + # requests + # yarl importlib-metadata==5.0.0 # via # flask # flytekit # keyring # mlflow +isodate==0.6.1 + # via azure-storage-blob itsdangerous==2.1.2 # via flask jaraco-classes==3.2.3 # via keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.2 # via # cookiecutter @@ -104,12 +198,18 @@ jinja2==3.1.2 # jinja2-time jinja2-time==0.2.0 # via cookiecutter +jmespath==1.0.1 + # via botocore joblib==1.2.0 # via flytekit keyring==23.11.0 # via flytekit +kubernetes==26.1.0 + # via flytekit mako==1.2.3 # via alembic +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.1 # via # jinja2 @@ -124,22 +224,38 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit +mdurl==0.1.2 + # via markdown-it-py mlflow==1.30.0 # via flytekitplugins-mlflow more-itertools==9.0.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==0.4.3 # via typing-inspect natsort==8.2.0 # via flytekit numpy==1.23.4 # via + # flytekit # mlflow # pandas # pyarrow # scipy oauthlib==3.2.2 - # via databricks-cli + # via + # databricks-cli + # requests-oauthlib packaging==21.3 # via # docker @@ -151,6 +267,8 @@ pandas==1.5.1 # mlflow plotly==5.11.0 # via flytekitplugins-mlflow +portalocker==2.7.0 + # via msal-extensions prometheus-client==0.15.0 # via prometheus-flask-exporter prometheus-flask-exporter==0.20.3 @@ -159,20 +277,29 @@ protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # mlflow # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==6.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pyjwt==2.6.0 - # via databricks-cli +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.6.0 + # via + # databricks-cli + # msal pyopenssl==22.1.0 # via flytekit pyparsing==3.0.9 @@ -180,8 +307,10 @@ pyparsing==3.0.9 python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit + # kubernetes # pandas python-json-logger==2.0.4 # via flytekit @@ -198,6 +327,7 @@ pyyaml==6.0 # via # cookiecutter # flytekit + # kubernetes # mlflow querystring-parser==1.2.4 # via mlflow @@ -205,22 +335,49 @@ regex==2022.10.31 # via docker-image-py requests==2.28.1 # via + # azure-core + # azure-datalake-store # cookiecutter # databricks-cli # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes # mlflow + # msal + # requests-oauthlib # responses +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes responses==0.22.0 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.6.0 # via flytekit scipy==1.9.3 # via mlflow +secretstorage==3.3.3 + # via keyring six==1.16.0 # via + # azure-core + # azure-identity # databricks-cli + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil # querystring-parser smmap==5.0.0 @@ -247,26 +404,37 @@ types-toml==0.10.8 # via responses typing-extensions==4.4.0 # via + # aioitertools + # azure-core + # azure-storage-blob # flytekit # typing-inspect typing-inspect==0.8.0 # via dataclasses-json urllib3==1.26.12 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses websocket-client==1.4.2 - # via docker + # via + # docker + # kubernetes werkzeug==2.2.2 # via flask wheel==0.38.3 # via flytekit wrapt==1.14.1 # via + # aiobotocore # deprecated # flytekit +yarl==1.9.2 + # via aiohttp zipp==3.10.0 # via importlib-metadata diff --git a/plugins/flytekit-mlflow/setup.py b/plugins/flytekit-mlflow/setup.py index 2033ce5d27..6d29b76ebd 100644 --- a/plugins/flytekit-mlflow/setup.py +++ b/plugins/flytekit-mlflow/setup.py @@ -4,7 +4,12 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0,<2.0.0", "plotly", "mlflow"] +plugin_requires = [ + "flytekit>=1.2.10,<1.3.0,<2.0.0", + "plotly", + "mlflow", + "flyteidl>=1.2.11,<1.3.0", +] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-modin/requirements.txt b/plugins/flytekit-modin/requirements.txt index c96c995db2..1b073fbc96 100644 --- a/plugins/flytekit-modin/requirements.txt +++ b/plugins/flytekit-modin/requirements.txt @@ -59,8 +59,10 @@ filelock==3.9.0 # via # ray # virtualenv -flyteidl==1.2.9 - # via flytekit +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-modin flytekit==1.2.7 # via flytekitplugins-modin frozenlist==1.3.3 diff --git a/plugins/flytekit-modin/setup.py b/plugins/flytekit-modin/setup.py index 3942af6e5c..2bd12fe1a8 100644 --- a/plugins/flytekit-modin/setup.py +++ b/plugins/flytekit-modin/setup.py @@ -11,6 +11,7 @@ "ray", "grpcio<=1.43.0", "grpcio-status<=1.43.0", + "flyteidl>=1.2.11,<1.3.0", ] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-onnx-pytorch/requirements.txt b/plugins/flytekit-onnx-pytorch/requirements.txt index 0660386414..6cd753e0c2 100644 --- a/plugins/flytekit-onnx-pytorch/requirements.txt +++ b/plugins/flytekit-onnx-pytorch/requirements.txt @@ -6,57 +6,145 @@ # -e file:.#egg=flytekitplugins-onnxpytorch # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit coloredlogs==15.0.1 # via onnxruntime -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 - # via retry -deprecated==1.2.13 + # via gcsfs +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flatbuffers==23.1.21 +flatbuffers==23.5.26 # via onnxruntime -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-onnxpytorch +flytekit==1.2.14 # via flytekitplugins-onnxpytorch -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -67,14 +155,20 @@ grpcio-status==1.48.2 humanfriendly==10.0 # via coloredlogs idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit # keyring + # rich-click importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jeepney==0.8.0 @@ -82,16 +176,18 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 + # via flytekit +keyring==24.1.1 # via flytekit -keyring==23.13.1 +kubernetes==26.1.0 # via flytekit -markupsafe==2.1.2 +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -102,13 +198,26 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes -mpmath==1.2.1 +mpmath==1.3.0 # via sympy +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit numpy==1.21.6 # via @@ -127,50 +236,65 @@ nvidia-cuda-runtime-cu11==11.7.99 # via torch nvidia-cudnn-cu11==8.5.0.96 # via torch -onnxruntime==1.14.0 +oauthlib==3.2.2 + # via requests-oauthlib +onnxruntime==1.14.1 # via -r requirements.in -packaging==23.0 +packaging==23.1 # via # docker # marshmallow # onnxruntime pandas==1.3.5 # via flytekit -pillow==9.4.0 +pillow==9.5.0 # via # -r requirements.in # torchvision +portalocker==2.7.0 + # via msal-extensions protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # onnxruntime # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -178,18 +302,40 @@ pyyaml==6.0 # via # cookiecutter # flytekit -regex==2022.10.31 + # kubernetes + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib # responses # torchvision -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.1.0 # via flytekit secretstorage==3.3.3 # via keyring @@ -197,8 +343,15 @@ singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 @@ -207,45 +360,61 @@ sympy==1.10.1 # via onnxruntime text-unidecode==1.3 # via python-slugify -toml==0.10.2 - # via responses torch==1.13.1 # via # flytekitplugins-onnxpytorch # torchvision torchvision==0.14.1 # via -r requirements.in -types-toml==0.10.8.5 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # importlib-metadata + # markdown-it-py + # pyjwt # responses + # rich # torch # torchvision # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via # flytekit # nvidia-cublas-cu11 # nvidia-cuda-runtime-cu11 -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources diff --git a/plugins/flytekit-onnx-pytorch/setup.py b/plugins/flytekit-onnx-pytorch/setup.py index be85288d6b..1298bef419 100644 --- a/plugins/flytekit-onnx-pytorch/setup.py +++ b/plugins/flytekit-onnx-pytorch/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "torch>=1.11.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "torch>=1.11.0", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-onnx-scikitlearn/requirements.txt b/plugins/flytekit-onnx-scikitlearn/requirements.txt index b2171351b6..7c76f29059 100644 --- a/plugins/flytekit-onnx-scikitlearn/requirements.txt +++ b/plugins/flytekit-onnx-scikitlearn/requirements.txt @@ -6,57 +6,145 @@ # -e file:.#egg=flytekitplugins-onnxscikitlearn # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit coloredlogs==15.0.1 # via onnxruntime -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 - # via retry -deprecated==1.2.13 + # via gcsfs +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flatbuffers==23.1.21 +flatbuffers==23.5.26 # via onnxruntime -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-onnxscikitlearn +flytekit==1.2.14 # via flytekitplugins-onnxscikitlearn -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -67,14 +155,20 @@ grpcio-status==1.48.2 humanfriendly==10.0 # via coloredlogs idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit # keyring + # rich-click importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jeepney==0.8.0 @@ -82,18 +176,20 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 # via # flytekit # scikit-learn -keyring==23.13.1 +keyring==24.1.1 # via flytekit -markupsafe==2.1.2 +kubernetes==26.1.0 + # via flytekit +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -104,13 +200,26 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes -mpmath==1.2.1 +mpmath==1.3.0 # via sympy +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit numpy==1.21.6 # via @@ -122,16 +231,17 @@ numpy==1.21.6 # pyarrow # scikit-learn # scipy - # skl2onnx -onnx==1.13.1 +oauthlib==3.2.2 + # via requests-oauthlib +onnx==1.14.0 # via # onnxconverter-common # skl2onnx onnxconverter-common==1.13.0 # via skl2onnx -onnxruntime==1.14.0 +onnxruntime==1.14.1 # via -r requirements.in -packaging==23.0 +packaging==23.1 # via # docker # marshmallow @@ -139,40 +249,52 @@ packaging==23.0 # onnxruntime pandas==1.3.5 # via flytekit +portalocker==2.7.0 + # via msal-extensions protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # onnx # onnxconverter-common # onnxruntime # protoc-gen-swagger - # skl2onnx protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -180,34 +302,61 @@ pyyaml==6.0 # via # cookiecutter # flytekit -regex==2022.10.31 + # kubernetes + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib # responses -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.1.0 # via flytekit scikit-learn==1.0.2 # via skl2onnx scipy==1.7.3 - # via - # scikit-learn - # skl2onnx + # via scikit-learn secretstorage==3.3.3 # via keyring singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil -skl2onnx==1.13 +skl2onnx==1.14.1 # via flytekitplugins-onnxscikitlearn +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 @@ -218,35 +367,54 @@ text-unidecode==1.3 # via python-slugify threadpoolctl==3.1.0 # via scikit-learn -toml==0.10.2 - # via responses -types-toml==0.10.8.5 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # importlib-metadata + # markdown-it-py # onnx + # pyjwt # responses + # rich # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/plugins/flytekit-onnx-scikitlearn/setup.py b/plugins/flytekit-onnx-scikitlearn/setup.py index eca2756f5b..859c518dc0 100644 --- a/plugins/flytekit-onnx-scikitlearn/setup.py +++ b/plugins/flytekit-onnx-scikitlearn/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "skl2onnx>=1.10.3"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "skl2onnx>=1.10.3", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-onnx-tensorflow/requirements.txt b/plugins/flytekit-onnx-tensorflow/requirements.txt index 33dddb5b25..9e2f71e378 100644 --- a/plugins/flytekit-onnx-tensorflow/requirements.txt +++ b/plugins/flytekit-onnx-tensorflow/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile requirements.in # @@ -37,7 +37,9 @@ cookiecutter==2.1.1 croniter==1.3.5 # via flytekit cryptography==37.0.4 - # via pyopenssl + # via + # pyopenssl + # secretstorage dataclasses-json==0.5.7 # via flytekit decorator==5.1.1 @@ -57,8 +59,10 @@ flatbuffers==1.12 # onnxruntime # tensorflow # tf2onnx -flyteidl==1.2.9 - # via flytekit +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-onnxtensorflow flytekit==1.2.7 # via flytekitplugins-onnxtensorflow gast==0.4.0 @@ -92,12 +96,18 @@ importlib-metadata==4.12.0 # flytekit # keyring # markdown +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.2 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter +joblib==1.3.1 + # via flytekit keras==2.9.0 # via tensorflow keras-preprocessing==1.1.2 @@ -125,6 +135,7 @@ natsort==8.1.0 # via flytekit numpy==1.23.0 # via + # flytekit # h5py # keras-preprocessing # onnx @@ -219,6 +230,8 @@ retry==0.9.2 # via flytekit rsa==4.8 # via google-auth +secretstorage==3.3.3 + # via keyring six==1.16.0 # via # astunparse diff --git a/plugins/flytekit-onnx-tensorflow/setup.py b/plugins/flytekit-onnx-tensorflow/setup.py index 6148cd8e1b..6407cfdf34 100644 --- a/plugins/flytekit-onnx-tensorflow/setup.py +++ b/plugins/flytekit-onnx-tensorflow/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "tf2onnx>=1.9.3", "tensorflow>=2.7.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "tf2onnx>=1.9.3", "tensorflow>=2.7.0", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt index f6c56bf14f..9a6008c7c7 100644 --- a/plugins/flytekit-pandera/requirements.txt +++ b/plugins/flytekit-pandera/requirements.txt @@ -6,53 +6,141 @@ # -e file:.#egg=flytekitplugins-pandera # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 - # via retry -deprecated==1.2.13 + # via gcsfs +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-pandera +flytekit==1.2.14 # via flytekitplugins-pandera -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -61,14 +149,21 @@ grpcio==1.48.2 grpcio-status==1.48.2 # via flytekit idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit # keyring + # rich-click + # typeguard importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jeepney==0.8.0 @@ -76,16 +171,18 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 # via flytekit -keyring==23.13.1 +keyring==24.1.1 # via flytekit -markupsafe==2.1.2 +kubernetes==26.1.0 + # via flytekit +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -96,11 +193,26 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl +multimethod==1.9.1 + # via pandera mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit numpy==1.21.6 # via @@ -108,7 +220,9 @@ numpy==1.21.6 # pandas # pandera # pyarrow -packaging==23.0 +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 # via # docker # marshmallow @@ -117,40 +231,53 @@ pandas==1.3.5 # via # flytekit # pandera -pandera==0.13.4 +pandera==0.15.2 # via flytekitplugins-pandera +portalocker==2.7.0 + # via msal-extensions protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pydantic==1.10.5 +pydantic==1.10.11 # via pandera -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -158,17 +285,39 @@ pyyaml==6.0 # via # cookiecutter # flytekit -regex==2022.10.31 + # kubernetes + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib # responses -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 + # via flytekit +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 # via flytekit -retry==0.9.2 +rsa==4.9 + # via google-auth +s3fs==2023.1.0 # via flytekit secretstorage==3.3.3 # via keyring @@ -176,47 +325,76 @@ singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -toml==0.10.2 - # via responses -types-toml==0.10.8.5 +typeguard==4.0.0 + # via pandera +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # importlib-metadata + # markdown-it-py # pandera # pydantic + # pyjwt # responses + # rich + # typeguard # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via # dataclasses-json # pandera -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit # pandera -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/plugins/flytekit-pandera/setup.py b/plugins/flytekit-pandera/setup.py index 049edf0e03..1a55b5e78e 100644 --- a/plugins/flytekit-pandera/setup.py +++ b/plugins/flytekit-pandera/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "pandera>=0.7.1"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "pandera>=0.7.1", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-papermill/dev-requirements.in b/plugins/flytekit-papermill/dev-requirements.in index 5285174ac3..6a730adfea 100644 --- a/plugins/flytekit-papermill/dev-requirements.in +++ b/plugins/flytekit-papermill/dev-requirements.in @@ -1,3 +1,3 @@ -flyteidl==1.2.9 +flyteidl==1.2.11 -e file:../../.#egg=flytekitplugins-pod&subdirectory=plugins/flytekit-k8s-pod -e file:../../.#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark diff --git a/plugins/flytekit-papermill/dev-requirements.txt b/plugins/flytekit-papermill/dev-requirements.txt index 0dfc2d57bb..7bd1b77222 100644 --- a/plugins/flytekit-papermill/dev-requirements.txt +++ b/plugins/flytekit-papermill/dev-requirements.txt @@ -52,10 +52,12 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==1.2.9 +flyteidl==1.2.11 # via # -r dev-requirements.in # flytekit + # flytekitplugins-pod + # flytekitplugins-spark flytekit==1.2.7 # via # flytekitplugins-pod diff --git a/plugins/flytekit-papermill/flytekitplugins/papermill/task.py b/plugins/flytekit-papermill/flytekitplugins/papermill/task.py index b1f472e99a..6f4ed6886c 100644 --- a/plugins/flytekit-papermill/flytekitplugins/papermill/task.py +++ b/plugins/flytekit-papermill/flytekitplugins/papermill/task.py @@ -133,6 +133,7 @@ def __init__( task_config: T = None, inputs: typing.Optional[typing.Dict[str, typing.Type]] = None, outputs: typing.Optional[typing.Dict[str, typing.Type]] = None, + output_notebooks: typing.Optional[bool] = True, **kwargs, ): # Each instance of NotebookTask instantiates an underlying task with a dummy function that will only be used @@ -165,13 +166,16 @@ def __init__( if not os.path.exists(self._notebook_path): raise ValueError(f"Illegal notebook path passed in {self._notebook_path}") - if outputs: + if output_notebooks: + if outputs is None: + outputs = {} outputs.update( { self._IMPLICIT_OP_NOTEBOOK: self._IMPLICIT_OP_NOTEBOOK_TYPE, self._IMPLICIT_RENDERED_NOTEBOOK: self._IMPLICIT_RENDERED_NOTEBOOK_TYPE, } ) + super().__init__( name, task_config, @@ -287,6 +291,8 @@ def execute(self, **kwargs) -> Any: else: raise TypeError(f"Expected output {k} of type {type_v} not found in the notebook outputs") + if len(output_list) == 1: + return output_list[0] return tuple(output_list) def post_execute(self, user_params: ExecutionParameters, rval: Any) -> Any: diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index 3964df44fa..3069f92e0d 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -1,63 +1,109 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.7 # by the following command: # # pip-compile requirements.in # -e file:.#egg=flytekitplugins-papermill # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp ansiwrap==0.8.4 # via papermill arrow==1.2.3 - # via jinja2-time -attrs==22.2.0 - # via jsonschema + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via + # aiohttp + # jsonschema +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs backcall==0.2.0 # via ipython -beautifulsoup4==4.11.2 +beautifulsoup4==4.12.2 # via nbconvert binaryornot==0.4.4 # via cookiecutter bleach==6.0.0 # via nbconvert -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit # papermill + # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit -debugpy==1.6.6 +debugpy==1.6.7 # via ipykernel decorator==5.1.1 # via + # gcsfs # ipython - # retry defusedxml==0.7.1 # via nbconvert -deprecated==1.2.13 +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit @@ -67,15 +113,57 @@ entrypoints==0.4 # via # jupyter-client # papermill -fastjsonschema==2.16.2 +fastjsonschema==2.17.1 # via nbformat -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-papermill +flytekit==1.2.14 # via flytekitplugins-papermill -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -84,12 +172,19 @@ grpcio==1.48.2 grpcio-status==1.48.2 # via flytekit idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs + # click # flytekit + # jsonschema # keyring # nbconvert + # nbformat + # rich-click importlib-resources==5.12.0 # via # jsonschema @@ -98,6 +193,8 @@ ipykernel==6.16.2 # via flytekitplugins-papermill ipython==7.34.0 # via ipykernel +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jedi==0.18.2 @@ -109,11 +206,10 @@ jeepney==0.8.0 jinja2==3.1.2 # via # cookiecutter - # jinja2-time # nbconvert -jinja2-time==0.2.0 - # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 # via flytekit jsonschema==4.17.3 # via nbformat @@ -129,9 +225,13 @@ jupyter-core==4.12.0 # nbformat jupyterlab-pygments==0.2.2 # via nbconvert -keyring==23.13.1 +keyring==24.1.1 # via flytekit -markupsafe==2.1.2 +kubernetes==26.1.0 + # via flytekit +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via # jinja2 # nbconvert @@ -148,21 +248,34 @@ matplotlib-inline==0.1.6 # via # ipykernel # ipython -mistune==2.0.5 +mdurl==0.1.2 + # via markdown-it-py +mistune==3.0.1 # via nbconvert -more-itertools==9.0.0 +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit -nbclient==0.7.2 +nbclient==0.7.4 # via # nbconvert # papermill -nbconvert==7.2.9 +nbconvert==7.6.0 # via flytekitplugins-papermill -nbformat==5.7.3 +nbformat==5.8.0 # via # nbclient # nbconvert @@ -176,7 +289,9 @@ numpy==1.21.6 # flytekit # pandas # pyarrow -packaging==23.0 +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 # via # docker # ipykernel @@ -196,49 +311,61 @@ pickleshare==0.7.5 # via ipython pkgutil-resolve-name==1.3.10 # via jsonschema -prompt-toolkit==3.0.37 +portalocker==2.7.0 + # via msal-extensions +prompt-toolkit==3.0.39 # via ipython protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -psutil==5.9.4 +psutil==5.9.5 # via ipykernel ptyprocess==0.7.0 # via pexpect -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pygments==2.14.0 +pygments==2.15.1 # via # ipython # nbconvert -pyopenssl==23.0.0 + # rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit pyrsistent==0.19.3 # via jsonschema python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit # jupyter-client + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -246,38 +373,69 @@ pyyaml==6.0 # via # cookiecutter # flytekit + # kubernetes # papermill -pyzmq==25.0.0 + # responses +pyzmq==25.1.0 # via # ipykernel # jupyter-client -regex==2022.10.31 +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal # papermill + # requests-oauthlib # responses -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 + # via flytekit +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 # via flytekit -retry==0.9.2 +rsa==4.9 + # via google-auth +s3fs==2023.1.0 # via flytekit secretstorage==3.3.3 # via keyring +singledispatchmethod==1.0 + # via flytekit six==1.16.0 # via + # azure-core + # azure-identity # bleach + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit -soupsieve==2.4 +soupsieve==2.4.1 # via beautifulsoup4 statsd==3.3.0 # via flytekit -tenacity==8.2.1 +tenacity==8.2.2 # via papermill text-unidecode==1.3 # via python-slugify @@ -285,13 +443,11 @@ textwrap3==0.9.2 # via ansiwrap tinycss2==1.2.1 # via nbconvert -toml==0.10.2 - # via responses tornado==6.2 # via # ipykernel # jupyter-client -tqdm==4.64.1 +tqdm==4.65.0 # via papermill traitlets==5.9.0 # via @@ -303,18 +459,35 @@ traitlets==5.9.0 # nbclient # nbconvert # nbformat -types-toml==0.10.8.5 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools + # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython + # importlib-metadata + # jsonschema + # markdown-it-py + # pyjwt + # responses + # rich # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses wcwidth==0.2.6 @@ -323,15 +496,20 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources diff --git a/plugins/flytekit-papermill/setup.py b/plugins/flytekit-papermill/setup.py index 32ebb0e49a..2fdc1ec3f0 100644 --- a/plugins/flytekit-papermill/setup.py +++ b/plugins/flytekit-papermill/setup.py @@ -9,6 +9,7 @@ "papermill>=1.2.0", "nbconvert>=6.0.7", "ipykernel>=5.0.0", + "flyteidl>=1.2.11,<1.3.0", ] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-papermill/tests/test_task.py b/plugins/flytekit-papermill/tests/test_task.py index 0e54e7082e..47db35793d 100644 --- a/plugins/flytekit-papermill/tests/test_task.py +++ b/plugins/flytekit-papermill/tests/test_task.py @@ -1,6 +1,7 @@ import datetime import os import tempfile +import typing import pandas as pd from flytekitplugins.papermill import NotebookTask @@ -8,7 +9,7 @@ from kubernetes.client import V1Container, V1PodSpec import flytekit -from flytekit import StructuredDataset, kwtypes, task +from flytekit import StructuredDataset, kwtypes, map_task, task, workflow from flytekit.configuration import Image, ImageConfig from flytekit.types.directory import FlyteDirectory from flytekit.types.file import FlyteFile, PythonNotebook @@ -33,6 +34,14 @@ def _get_nb_path(name: str, suffix: str = "", abs: bool = True, ext: str = ".ipy outputs=kwtypes(square=float), ) +nb_sub_task = NotebookTask( + name="test", + notebook_path=_get_nb_path(nb_name, abs=False), + inputs=kwtypes(a=float), + outputs=kwtypes(square=float), + output_notebooks=False, +) + def test_notebook_task_simple(): serialization_settings = flytekit.configuration.SerializationSettings( @@ -172,3 +181,11 @@ def create_sd() -> StructuredDataset: ) success, out, render = nb_types.execute(ff=ff, fd=fd, sd=sd) assert success is True, "Notebook execution failed" + + +def test_map_over_notebook_task(): + @workflow + def wf(a: float) -> typing.List[float]: + return map_task(nb_sub_task)(a=[a, a]) + + assert wf(a=3.14) == [9.8596, 9.8596] diff --git a/plugins/flytekit-polars/requirements.txt b/plugins/flytekit-polars/requirements.txt index 9deb7d3482..3ba8e7f4c6 100644 --- a/plugins/flytekit-polars/requirements.txt +++ b/plugins/flytekit-polars/requirements.txt @@ -6,53 +6,141 @@ # -e file:.#egg=flytekitplugins-polars # via -r requirements.in +adlfs==2022.2.0 + # via flytekit +aiobotocore==2.4.2 + # via s3fs +aiohttp==3.8.4 + # via + # adlfs + # aiobotocore + # gcsfs + # s3fs +aioitertools==0.11.0 + # via aiobotocore +aiosignal==1.3.1 + # via aiohttp arrow==1.2.3 - # via jinja2-time + # via cookiecutter +async-timeout==4.0.2 + # via aiohttp +asynctest==0.13.0 + # via aiohttp +attrs==23.1.0 + # via aiohttp +azure-core==1.28.0 + # via + # adlfs + # azure-identity + # azure-storage-blob +azure-datalake-store==0.0.53 + # via adlfs +azure-identity==1.13.0 + # via adlfs +azure-storage-blob==12.16.0 + # via adlfs binaryornot==0.4.4 # via cookiecutter -certifi==2022.12.7 - # via requests +botocore==1.27.59 + # via aiobotocore +cachetools==5.3.1 + # via google-auth +certifi==2023.5.7 + # via + # kubernetes + # requests cffi==1.15.1 - # via cryptography + # via + # azure-datalake-store + # cryptography chardet==5.1.0 # via binaryornot -charset-normalizer==3.0.1 - # via requests -click==8.1.3 +charset-normalizer==3.2.0 + # via + # aiohttp + # requests +click==8.1.4 # via # cookiecutter # flytekit + # rich-click cloudpickle==2.2.1 # via flytekit -cookiecutter==2.1.1 +cookiecutter==2.2.2 # via flytekit -croniter==1.3.8 +croniter==1.4.1 # via flytekit -cryptography==39.0.1 +cryptography==41.0.1 # via + # azure-identity + # azure-storage-blob + # msal + # pyjwt # pyopenssl # secretstorage -dataclasses-json==0.5.7 +dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 - # via retry -deprecated==1.2.13 + # via gcsfs +deprecated==1.2.14 # via flytekit -diskcache==5.4.0 +diskcache==5.6.1 # via flytekit -docker==6.0.1 +docker==6.1.3 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.9 - # via flytekit -flytekit==1.2.7 +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-polars +flytekit==1.2.14 # via flytekitplugins-polars -googleapis-common-protos==1.58.0 +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2023.1.0 + # via + # adlfs + # flytekit + # gcsfs + # s3fs +gcsfs==2023.1.0 + # via flytekit +gitdb==4.0.10 + # via gitpython +gitpython==3.1.32 + # via flytekit +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.21.0 + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # kubernetes +google-auth-oauthlib==1.0.0 + # via gcsfs +google-cloud-core==2.3.3 + # via google-cloud-storage +google-cloud-storage==2.10.0 + # via gcsfs +google-crc32c==1.5.0 + # via google-resumable-media +google-resumable-media==2.5.0 + # via google-cloud-storage +googleapis-common-protos==1.59.1 # via # flyteidl + # flytekit + # google-api-core # grpcio-status grpcio==1.48.2 # via @@ -61,14 +149,20 @@ grpcio==1.48.2 grpcio-status==1.48.2 # via flytekit idna==3.4 - # via requests -importlib-metadata==6.0.0 # via + # requests + # yarl +importlib-metadata==6.7.0 + # via + # attrs # click # flytekit # keyring + # rich-click importlib-resources==5.12.0 # via keyring +isodate==0.6.1 + # via azure-storage-blob jaraco-classes==3.2.3 # via keyring jeepney==0.8.0 @@ -76,16 +170,18 @@ jeepney==0.8.0 # keyring # secretstorage jinja2==3.1.2 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 # via cookiecutter -joblib==1.2.0 +jmespath==1.0.1 + # via botocore +joblib==1.3.1 + # via flytekit +keyring==24.1.1 # via flytekit -keyring==23.13.1 +kubernetes==26.1.0 # via flytekit -markupsafe==2.1.2 +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.3 # via jinja2 marshmallow==3.19.0 # via @@ -96,55 +192,83 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -more-itertools==9.0.0 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 # via jaraco-classes +msal==1.22.0 + # via + # azure-datalake-store + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +multidict==6.0.4 + # via + # aiohttp + # yarl mypy-extensions==1.0.0 # via typing-inspect -natsort==8.2.0 +natsort==8.4.0 # via flytekit numpy==1.21.6 # via # flytekit # pandas # pyarrow -packaging==23.0 +oauthlib==3.2.2 + # via requests-oauthlib +packaging==23.1 # via # docker # marshmallow pandas==1.3.5 # via flytekit -polars==0.16.8 +polars==0.18.4 # via flytekitplugins-polars +portalocker==2.7.0 + # via msal-extensions protobuf==3.20.3 # via # flyteidl # flytekit + # google-api-core # googleapis-common-protos # grpcio-status # protoc-gen-swagger protoc-gen-swagger==0.1.0 # via flyteidl -py==1.11.0 - # via retry pyarrow==10.0.1 # via flytekit +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth pycparser==2.21 # via cffi -pyopenssl==23.0.0 +pygments==2.15.1 + # via rich +pyjwt[crypto]==2.7.0 + # via msal +pyopenssl==23.2.0 # via flytekit python-dateutil==2.8.2 # via # arrow + # botocore # croniter # flytekit + # kubernetes # pandas python-json-logger==2.0.7 # via flytekit -python-slugify==8.0.0 +python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.7.1 +pytz==2023.3 # via # flytekit # pandas @@ -152,17 +276,39 @@ pyyaml==6.0 # via # cookiecutter # flytekit -regex==2022.10.31 + # kubernetes + # responses +regex==2023.6.3 # via docker-image-py -requests==2.28.2 +requests==2.31.0 # via + # azure-core + # azure-datalake-store # cookiecutter # docker # flytekit + # gcsfs + # google-api-core + # google-cloud-storage + # kubernetes + # msal + # requests-oauthlib # responses -responses==0.22.0 +requests-oauthlib==1.3.1 + # via + # google-auth-oauthlib + # kubernetes +responses==0.23.1 # via flytekit -retry==0.9.2 +rich==13.4.2 + # via + # flytekit + # rich-click +rich-click==1.6.1 + # via flytekit +rsa==4.9 + # via google-auth +s3fs==2023.1.0 # via flytekit secretstorage==3.3.3 # via keyring @@ -170,43 +316,69 @@ singledispatchmethod==1.0 # via flytekit six==1.16.0 # via + # azure-core + # azure-identity + # google-auth # grpcio + # isodate + # kubernetes # python-dateutil +smmap==5.0.0 + # via gitdb sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -toml==0.10.2 - # via responses -types-toml==0.10.8.5 +types-pyyaml==6.0.12.10 # via responses -typing-extensions==4.5.0 +typing-extensions==4.7.1 # via + # aiohttp + # aioitertools # arrow + # async-timeout + # azure-core + # azure-storage-blob # flytekit + # gitpython # importlib-metadata + # markdown-it-py # polars + # pyjwt # responses + # rich # typing-inspect -typing-inspect==0.8.0 + # yarl +typing-inspect==0.9.0 # via dataclasses-json -urllib3==1.26.14 +urllib3==1.26.16 # via + # botocore # docker # flytekit + # google-auth + # kubernetes # requests # responses -websocket-client==1.5.1 - # via docker -wheel==0.38.4 +websocket-client==1.6.1 + # via + # docker + # kubernetes +wheel==0.40.0 # via flytekit -wrapt==1.14.1 +wrapt==1.15.0 # via + # aiobotocore # deprecated # flytekit -zipp==3.14.0 +yarl==1.9.2 + # via aiohttp +zipp==3.15.0 # via # importlib-metadata # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/plugins/flytekit-polars/setup.py b/plugins/flytekit-polars/setup.py index 6e55f217a5..f7fc826ddc 100644 --- a/plugins/flytekit-polars/setup.py +++ b/plugins/flytekit-polars/setup.py @@ -7,6 +7,7 @@ plugin_requires = [ "flytekit>=1.1.0b0,<1.3.0,<2.0.0", "polars>=0.8.27", + "flyteidl>=1.2.11,<1.3.0", ] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-ray/requirements.txt b/plugins/flytekit-ray/requirements.txt index 5f81a1059e..546000747a 100644 --- a/plugins/flytekit-ray/requirements.txt +++ b/plugins/flytekit-ray/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile requirements.in # @@ -55,7 +55,9 @@ cookiecutter==1.7.3 croniter==1.3.5 # via flytekit cryptography==37.0.2 - # via pyopenssl + # via + # pyopenssl + # secretstorage dataclasses-json==0.5.7 # via flytekit decorator==5.1.1 @@ -76,7 +78,7 @@ filelock==3.7.1 # via # ray # virtualenv -flyteidl==1.2.9 +flyteidl==1.2.11 # via # flytekit # flytekitplugins-ray @@ -113,12 +115,20 @@ importlib-metadata==4.11.3 # via # flytekit # keyring +importlib-resources==6.0.0 + # via jsonschema +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.2 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter +joblib==1.3.1 + # via flytekit jsonschema==4.6.1 # via ray keyring==23.5.0 @@ -146,6 +156,7 @@ natsort==8.1.0 # via flytekit numpy==1.21.6 # via + # flytekit # pandas # pyarrow # ray @@ -236,6 +247,8 @@ retry==0.9.2 # via flytekit rsa==4.9 # via google-auth +secretstorage==3.3.3 + # via keyring six==1.16.0 # via # blessed @@ -279,7 +292,9 @@ wrapt==1.14.1 yarl==1.8.2 # via aiohttp zipp==3.8.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/plugins/flytekit-ray/setup.py b/plugins/flytekit-ray/setup.py index a83b0822fc..71000f9649 100644 --- a/plugins/flytekit-ray/setup.py +++ b/plugins/flytekit-ray/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["ray[default]", "flytekit>=1.1.0b0,<1.3.0,<2.0.0", "flyteidl>=1.1.10"] +plugin_requires = ["ray[default]", "flytekit>=1.1.0b0,<1.3.0,<2.0.0", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt index 9e44535ee0..1db9913cfa 100644 --- a/plugins/flytekit-snowflake/requirements.txt +++ b/plugins/flytekit-snowflake/requirements.txt @@ -46,8 +46,10 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.9 - # via flytekit +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-snowflake flytekit==1.2.7 # via flytekitplugins-snowflake googleapis-common-protos==1.58.0 diff --git a/plugins/flytekit-snowflake/setup.py b/plugins/flytekit-snowflake/setup.py index 78f640c9fb..b8ec8dcbfc 100644 --- a/plugins/flytekit-snowflake/setup.py +++ b/plugins/flytekit-snowflake/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-spark/flytekitplugins/spark/task.py b/plugins/flytekit-spark/flytekitplugins/spark/task.py index 564e55778f..0d8ecd5b6e 100644 --- a/plugins/flytekit-spark/flytekitplugins/spark/task.py +++ b/plugins/flytekit-spark/flytekitplugins/spark/task.py @@ -23,10 +23,14 @@ class Spark(object): Args: spark_conf: Dictionary of spark config. The variables should match what spark expects hadoop_conf: Dictionary of hadoop conf. The variables should match a typical hadoop configuration for spark + executor_path: Python binary executable to use for PySpark in driver and executor. + applications_path: MainFile is the path to a bundled JAR, Python, or R file of the application to execute. """ spark_conf: Optional[Dict[str, str]] = None hadoop_conf: Optional[Dict[str, str]] = None + executor_path: Optional[str] = None + applications_path: Optional[str] = None def __post_init__(self): if self.spark_conf is None: @@ -107,8 +111,8 @@ def __init__( **kwargs, ): self.sess: Optional[SparkSession] = None - self._default_executor_path: Optional[str] = None - self._default_applications_path: Optional[str] = None + self._default_executor_path: Optional[str] = task_config.executor_path + self._default_applications_path: Optional[str] = task_config.applications_path if isinstance(container_image, ImageSpec): if container_image.base_image is None: diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt index c30b61fe55..83a03af1b7 100644 --- a/plugins/flytekit-spark/requirements.txt +++ b/plugins/flytekit-spark/requirements.txt @@ -46,8 +46,10 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.9 - # via flytekit +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-spark flytekit==1.2.7 # via flytekitplugins-spark googleapis-common-protos==1.58.0 diff --git a/plugins/flytekit-spark/setup.py b/plugins/flytekit-spark/setup.py index 11935a30af..b2a2f2ca61 100644 --- a/plugins/flytekit-spark/setup.py +++ b/plugins/flytekit-spark/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "pyspark>=3.0.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "pyspark>=3.0.0", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-spark/tests/test_spark_task.py b/plugins/flytekit-spark/tests/test_spark_task.py index 7049684a2d..f4f8422c38 100644 --- a/plugins/flytekit-spark/tests/test_spark_task.py +++ b/plugins/flytekit-spark/tests/test_spark_task.py @@ -36,7 +36,13 @@ def test_spark_task(reset_spark_session): }, } - @task(task_config=Spark(spark_conf={"spark": "1"})) + @task( + task_config=Spark( + spark_conf={"spark": "1"}, + executor_path="/usr/bin/python3", + applications_path="local:///usr/local/bin/entrypoint.py", + ) + ) def my_spark(a: str) -> int: session = flytekit.current_context().spark_session assert session.sparkContext.appName == "FlyteSpark: ex:local:local:local" @@ -56,6 +62,8 @@ def my_spark(a: str) -> int: retrieved_settings = my_spark.get_custom(settings) assert retrieved_settings["sparkConf"] == {"spark": "1"} + assert retrieved_settings["executorPath"] == "/usr/bin/python3" + assert retrieved_settings["mainApplicationFile"] == "local:///usr/local/bin/entrypoint.py" pb = ExecutionParameters.new_builder() pb.working_dir = "/tmp" diff --git a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/task.py b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/task.py index 88e4ef41c0..7b3d6e6ef2 100644 --- a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/task.py +++ b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/task.py @@ -11,6 +11,7 @@ from flytekit.core.base_sql_task import SQLTask from flytekit.core.python_customized_container_task import PythonCustomizedContainerTask from flytekit.core.shim_task import ShimTaskExecutor +from flytekit.loggers import logger from flytekit.models import task as task_models from flytekit.models.security import Secret from flytekit.types.schema import FlyteSchema @@ -126,10 +127,10 @@ def execute_from_model(self, tt: task_models.TaskTemplate, **kwargs) -> typing.A tt.custom["connect_args"][key] = value engine = create_engine(tt.custom["uri"], connect_args=tt.custom["connect_args"], echo=False) - print(f"Connecting to db {tt.custom['uri']}") + logger.info(f"Connecting to db {tt.custom['uri']}") interpolated_query = SQLAlchemyTask.interpolate_query(tt.custom["query_template"], **kwargs) - print(f"Interpolated query {interpolated_query}") + logger.info(f"Interpolated query {interpolated_query}") with engine.begin() as connection: df = None if tt.interface.outputs: diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt index d79a8ff148..7f555fd7e0 100644 --- a/plugins/flytekit-sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.7 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile requirements.in @@ -46,8 +46,10 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.15 # via flytekit -flyteidl==1.2.9 - # via flytekit +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-sqlalchemy flytekit==1.2.7 # via flytekitplugins-sqlalchemy googleapis-common-protos==1.58.0 @@ -66,10 +68,8 @@ idna==3.4 # via requests importlib-metadata==6.0.0 # via - # click # flytekit # keyring - # sqlalchemy importlib-resources==5.12.0 # via keyring jaraco-classes==3.2.3 @@ -167,8 +167,6 @@ retry==0.9.2 # via flytekit secretstorage==3.3.3 # via keyring -singledispatchmethod==1.0 - # via flytekit six==1.16.0 # via # grpcio @@ -176,9 +174,7 @@ six==1.16.0 sortedcontainers==2.4.0 # via flytekit sqlalchemy==1.4.39 - # via - # -r requirements.in - # flytekitplugins-sqlalchemy + # via flytekitplugins-sqlalchemy statsd==3.3.0 # via flytekit text-unidecode==1.3 @@ -189,10 +185,7 @@ types-toml==0.10.8.5 # via responses typing-extensions==4.5.0 # via - # arrow # flytekit - # importlib-metadata - # responses # typing-inspect typing-inspect==0.8.0 # via dataclasses-json diff --git a/plugins/flytekit-sqlalchemy/setup.py b/plugins/flytekit-sqlalchemy/setup.py index 46448fb49d..ba30c12b01 100644 --- a/plugins/flytekit-sqlalchemy/setup.py +++ b/plugins/flytekit-sqlalchemy/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "sqlalchemy>=1.4.7"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "sqlalchemy>=1.4.7", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-vaex/requirements.txt b/plugins/flytekit-vaex/requirements.txt index 9c8a1c7009..3c6a12dafa 100644 --- a/plugins/flytekit-vaex/requirements.txt +++ b/plugins/flytekit-vaex/requirements.txt @@ -57,8 +57,10 @@ docstring-parser==0.15 # via flytekit filelock==3.9.0 # via vaex-core -flyteidl==1.2.9 - # via flytekit +flyteidl==1.2.11 + # via + # flytekit + # flytekitplugins-vaex flytekit==1.2.7 # via flytekitplugins-vaex frozendict==2.3.5 diff --git a/plugins/flytekit-vaex/setup.py b/plugins/flytekit-vaex/setup.py index cc55a5d235..415fe54b2a 100644 --- a/plugins/flytekit-vaex/setup.py +++ b/plugins/flytekit-vaex/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "vaex-core>=4.13.0,<4.14"] +plugin_requires = ["flytekit>=1.1.0b0,<1.3.0,<2.0.0", "vaex-core>=4.13.0,<4.14", "flyteidl>=1.2.11,<1.3.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-whylogs/requirements.txt b/plugins/flytekit-whylogs/requirements.txt index 8dc1500e09..0a192b3b5d 100644 --- a/plugins/flytekit-whylogs/requirements.txt +++ b/plugins/flytekit-whylogs/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.7 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile requirements.in @@ -8,16 +8,18 @@ # via -r requirements.in backcall==0.2.0 # via ipython -certifi==2022.12.7 +certifi==2023.5.7 # via requests -charset-normalizer==3.0.1 +charset-normalizer==3.2.0 # via requests decorator==5.1.1 # via ipython +flyteidl==1.2.11 + # via flytekitplugins-whylogs +googleapis-common-protos==1.59.1 + # via flyteidl idna==3.4 # via requests -importlib-metadata==4.2.0 - # via whylogs ipython==7.34.0 # via whylogs jedi==0.18.2 @@ -34,25 +36,30 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pillow==9.4.0 +pillow==9.5.0 # via whylogs -prompt-toolkit==3.0.37 +prompt-toolkit==3.0.39 # via ipython protobuf==3.20.3 # via + # flyteidl # flytekitplugins-whylogs + # googleapis-common-protos + # protoc-gen-swagger # whylogs +protoc-gen-swagger==0.1.0 + # via flyteidl ptyprocess==0.7.0 # via pexpect pybars3==0.9.7 # via whylogs -pygments==2.14.0 +pygments==2.15.1 # via ipython pymeta3==0.5.1 # via pybars3 python-dateutil==2.8.2 # via whylabs-client -requests==2.28.2 +requests==2.31.0 # via whylogs scipy==1.7.3 # via whylogs @@ -62,24 +69,22 @@ traitlets==5.9.0 # via # ipython # matplotlib-inline -typing-extensions==4.5.0 - # via - # importlib-metadata - # whylogs -urllib3==1.26.14 +typing-extensions==4.7.1 + # via whylogs +urllib3==2.0.3 # via # requests # whylabs-client wcwidth==0.2.6 # via prompt-toolkit -whylabs-client==0.4.2 +whylabs-client==0.5.2 # via whylogs whylogs[viz]==1.1.27 - # via flytekitplugins-whylogs + # via + # -r requirements.in + # flytekitplugins-whylogs whylogs-sketching==3.4.1.dev3 # via whylogs -zipp==3.14.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/plugins/flytekit-whylogs/setup.py b/plugins/flytekit-whylogs/setup.py index ce10e877f6..45603c51ba 100644 --- a/plugins/flytekit-whylogs/setup.py +++ b/plugins/flytekit-whylogs/setup.py @@ -4,7 +4,11 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["protobuf>=3.15,<4.0.0", "whylogs[viz]>=1.0.8"] +plugin_requires = [ + "protobuf>=3.15,<4.0.0", + "whylogs[viz]>=1.0.8", + "flyteidl>=1.2.11,<1.3.0", +] __version__ = "0.0.0+develop" diff --git a/setup.py b/setup.py index 1c50883aa5..8af2623047 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,6 @@ "marshmallow-jsonschema>=0.12.0", "natsort>=7.0.1", "docker-image-py>=0.1.10", - "singledispatchmethod; python_version < '3.8.0'", "typing_extensions", "docstring-parser>=0.9.0", "diskcache>=5.2.1", diff --git a/tests/flytekit/integration/remote/test_remote.py b/tests/flytekit/integration/remote/test_remote.py index 78a828a507..3466a48d92 100644 --- a/tests/flytekit/integration/remote/test_remote.py +++ b/tests/flytekit/integration/remote/test_remote.py @@ -172,7 +172,9 @@ def test_execute_python_task(flyteclient, flyte_workflows_register, flyte_remote t1._name = t1.name.replace("mock_flyte_repo.", "") remote = FlyteRemote(Config.auto(), PROJECT, "development") - execution = remote.execute(t1, inputs={"a": 10}, version=f"v{VERSION}", wait=True) + execution = remote.execute( + t1, inputs={"a": 10}, version=f"v{VERSION}", wait=True, overwrite_cache=True, envs={"foo": "bar"} + ) assert execution.outputs["t1_int_output"] == 12 assert execution.outputs["c"] == "world" diff --git a/tests/flytekit/unit/cli/pyflyte/test_run.py b/tests/flytekit/unit/cli/pyflyte/test_run.py index 1e2ad98fd3..6629eb245b 100644 --- a/tests/flytekit/unit/cli/pyflyte/test_run.py +++ b/tests/flytekit/unit/cli/pyflyte/test_run.py @@ -128,6 +128,8 @@ def test_pyflyte_run_cli(): json.dumps([{"x": parquet_file}]), "--o", json.dumps({"x": [parquet_file]}), + "--p", + "Any", ], catch_exceptions=False, ) @@ -162,15 +164,10 @@ def test_union_type1(input): ) def test_union_type2(input): runner = CliRunner() + env = '{"foo": "bar"}' result = runner.invoke( pyflyte.main, - [ - "run", - os.path.join(DIR_NAME, "workflow.py"), - "test_union2", - "--a", - input, - ], + ["run", "--overwrite-cache", "--envs", env, os.path.join(DIR_NAME, "workflow.py"), "test_union2", "--a", input], catch_exceptions=False, ) print(result.stdout) @@ -272,9 +269,9 @@ def test_list_default_arguments(wf_path): ) ic_result_4 = ImageConfig( - default_image=Image(name="default", fqn="flytekit", tag="4VC-c-UDrUvfySJ0aS3qCw.."), + default_image=Image(name="default", fqn="flytekit", tag="mMxGzKCqxVk8msz0yV22-g.."), images=[ - Image(name="default", fqn="flytekit", tag="4VC-c-UDrUvfySJ0aS3qCw.."), + Image(name="default", fqn="flytekit", tag="mMxGzKCqxVk8msz0yV22-g.."), Image(name="xyz", fqn="docker.io/xyz", tag="latest"), Image(name="abc", fqn="docker.io/abc", tag=None), ], diff --git a/tests/flytekit/unit/cli/pyflyte/workflow.py b/tests/flytekit/unit/cli/pyflyte/workflow.py index 01621a6a01..311f141a22 100644 --- a/tests/flytekit/unit/cli/pyflyte/workflow.py +++ b/tests/flytekit/unit/cli/pyflyte/workflow.py @@ -58,8 +58,9 @@ def print_all( m: dict, n: typing.List[typing.Dict[str, FlyteFile]], o: typing.Dict[str, typing.List[FlyteFile]], + p: typing.Any, ): - print(f"{a}, {b}, {c}, {d}, {e}, {f}, {g}, {h}, {i}, {j}, {k}, {l}, {m}, {n}, {o}") + print(f"{a}, {b}, {c}, {d}, {e}, {f}, {g}, {h}, {i}, {j}, {k}, {l}, {m}, {n}, {o} , {p}") @task @@ -88,6 +89,7 @@ def my_wf( l: dict, n: typing.List[typing.Dict[str, FlyteFile]], o: typing.Dict[str, typing.List[FlyteFile]], + p: typing.Any, remote: pd.DataFrame, image: StructuredDataset, m: dict = {"hello": "world"}, @@ -95,5 +97,5 @@ def my_wf( x = get_subset_df(df=remote) # noqa: shown for demonstration; users should use the same types between tasks show_sd(in_sd=x) show_sd(in_sd=image) - print_all(a=a, b=b, c=c, d=d, e=e, f=f, g=g, h=h, i=i, j=j, k=k, l=l, m=m, n=n, o=o) + print_all(a=a, b=b, c=c, d=d, e=e, f=f, g=g, h=h, i=i, j=j, k=k, l=l, m=m, n=n, o=o, p=p) return x diff --git a/tests/flytekit/unit/core/image_spec/test_image_spec.py b/tests/flytekit/unit/core/image_spec/test_image_spec.py index be8ea61427..46b5cb0244 100644 --- a/tests/flytekit/unit/core/image_spec/test_image_spec.py +++ b/tests/flytekit/unit/core/image_spec/test_image_spec.py @@ -27,7 +27,7 @@ def test_image_spec(): assert image_spec.source_root is None assert image_spec.env is None assert image_spec.is_container() is True - assert image_spec.image_name() == "flytekit:yZ8jICcDTLoDArmNHbWNwg.." + assert image_spec.image_name() == "flytekit:OLFSrRjcG5_uXuRqd0TSdQ.." ctx = context_manager.FlyteContext.current_context() with context_manager.FlyteContextManager.with_context( ctx.with_execution_state(ctx.execution_state.with_params(mode=ExecutionState.Mode.TASK_EXECUTION)) @@ -42,7 +42,7 @@ def build_image(self, img): ImageBuildEngine.register("dummy", DummyImageSpecBuilder()) ImageBuildEngine._REGISTRY["dummy"].build_image(image_spec) assert "dummy" in ImageBuildEngine._REGISTRY - assert calculate_hash_from_image_spec(image_spec) == "yZ8jICcDTLoDArmNHbWNwg.." + assert calculate_hash_from_image_spec(image_spec) == "OLFSrRjcG5_uXuRqd0TSdQ.." assert image_spec.exist() is False with pytest.raises(Exception): diff --git a/tests/flytekit/unit/core/test_flyte_file.py b/tests/flytekit/unit/core/test_flyte_file.py index b7f0a1aeee..5dd05cdffd 100644 --- a/tests/flytekit/unit/core/test_flyte_file.py +++ b/tests/flytekit/unit/core/test_flyte_file.py @@ -439,13 +439,20 @@ def test_flyte_file_annotated_hashmethod(local_dummy_file): def calc_hash(ff: FlyteFile) -> str: return str(ff.path) + HashedFlyteFile = Annotated[FlyteFile, HashMethod(calc_hash)] + @task - def t1(path: str) -> Annotated[FlyteFile, HashMethod(calc_hash)]: - return FlyteFile(path) + def t1(path: str) -> HashedFlyteFile: + return HashedFlyteFile(path) + + @task + def t2(ff: HashedFlyteFile) -> None: + print(ff.path) @workflow def wf(path: str) -> None: - t1(path=path) + ff = t1(path=path) + t2(ff=ff) wf(path=local_dummy_file) diff --git a/tests/flytekit/unit/core/test_node_creation.py b/tests/flytekit/unit/core/test_node_creation.py index 0fb9d6677c..8e731f02dc 100644 --- a/tests/flytekit/unit/core/test_node_creation.py +++ b/tests/flytekit/unit/core/test_node_creation.py @@ -456,3 +456,16 @@ def my_wf(a: str) -> str: return t1(a=a).with_overrides(task_config=None) my_wf() + + +def test_override_image(): + @task + def bar(): + print("hello") + + @workflow + def wf() -> str: + bar().with_overrides(container_image="hello/world") + return "hi" + + assert wf.nodes[0].flyte_entity.container_image == "hello/world" diff --git a/tests/flytekit/unit/core/test_promise.py b/tests/flytekit/unit/core/test_promise.py index 9478cc33ba..b214764c50 100644 --- a/tests/flytekit/unit/core/test_promise.py +++ b/tests/flytekit/unit/core/test_promise.py @@ -145,3 +145,18 @@ def t1(a: typing.Union[float, typing.Dict[str, int]]): t1.interface.inputs, t1.python_interface.inputs, ) + + +def test_optional_task_kwargs(): + from typing import Optional + + from flytekit import Workflow + + @task + def func(foo: Optional[int] = None): + pass + + wf = Workflow(name="test") + wf.add_entity(func, foo=None) + + wf() diff --git a/tests/flytekit/unit/core/test_python_function_task.py b/tests/flytekit/unit/core/test_python_function_task.py index 160efb0504..5e6872d43a 100644 --- a/tests/flytekit/unit/core/test_python_function_task.py +++ b/tests/flytekit/unit/core/test_python_function_task.py @@ -75,7 +75,7 @@ def build_image(self, img): ImageBuildEngine.register("test", TestImageSpecBuilder()) assert ( get_registerable_container_image(ImageSpec(builder="test", python_version="3.7"), cfg) - == "flytekit:0N8X-XowtpEkDYWDlb8Abg.." + == "flytekit:usEl-DNx_srVn7zp2vHlBw.." ) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index a0aafc3c13..619be3b932 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -41,6 +41,8 @@ UnionTransformer, convert_json_schema_to_python_class, dataclass_from_dict, + get_underlying_type, + is_annotated, ) from flytekit.exceptions import user as user_exceptions from flytekit.models import types as model_types @@ -1685,3 +1687,28 @@ def test_batch_pickle_list(python_val, python_type, expected_list_length): # data = task0() # task0() -> Annotated[typing.List[FlytePickle], BatchSize(2)] # task1(data=data) # task1(data: typing.List[FlytePickle]) assert pv == python_val + + +@pytest.mark.parametrize( + "t,expected", + [ + (list, False), + (Annotated[int, "tag"], True), + (Annotated[typing.List[str], "a", "b"], True), + (Annotated[typing.Dict[int, str], FlyteAnnotation({"foo": "bar"})], True), + ], +) +def test_is_annotated(t, expected): + assert is_annotated(t) == expected + + +@pytest.mark.parametrize( + "t,expected", + [ + (typing.List, typing.List), + (Annotated[int, "tag"], int), + (Annotated[typing.List[str], "a", "b"], typing.List[str]), + ], +) +def test_get_underlying_type(t, expected): + assert get_underlying_type(t) == expected diff --git a/tests/flytekit/unit/deck/test_deck.py b/tests/flytekit/unit/deck/test_deck.py index f65c94b877..ebba6d23dc 100644 --- a/tests/flytekit/unit/deck/test_deck.py +++ b/tests/flytekit/unit/deck/test_deck.py @@ -91,7 +91,7 @@ def t_df(a: str) -> int: assert len(ctx.user_space_params.decks) == expected_decks -@mock.patch("flytekit.deck.deck._ipython_check") +@mock.patch("flytekit.deck.deck.ipython_check") def test_deck_in_jupyter(mock_ipython_check): mock_ipython_check.return_value = True diff --git a/tests/flytekit/unit/remote/test_remote.py b/tests/flytekit/unit/remote/test_remote.py index 5bfd7e4bf6..94b03b044a 100644 --- a/tests/flytekit/unit/remote/test_remote.py +++ b/tests/flytekit/unit/remote/test_remote.py @@ -1,17 +1,22 @@ import os import pathlib import tempfile +import typing from collections import OrderedDict from datetime import datetime, timedelta +import mock import pytest from flyteidl.core import compiler_pb2 as _compiler_pb2 +from flyteidl.service import dataproxy_pb2 from mock import MagicMock, patch import flytekit.configuration from flytekit import CronSchedule, LaunchPlan, task, workflow from flytekit.configuration import Config, DefaultImages, ImageConfig from flytekit.core.base_task import PythonTask +from flytekit.core.context_manager import FlyteContextManager +from flytekit.core.type_engine import TypeEngine from flytekit.exceptions import user as user_exceptions from flytekit.models import common as common_models from flytekit.models import security @@ -338,3 +343,21 @@ def test_launch_backfill(remote): wf = remote.launch_backfill("p", "d", start_date, end_date, "daily2", "v1", dry_run=True) assert wf + + +@mock.patch("flytekit.remote.remote.FlyteRemote.client") +def test_local_server(mock_client): + ctx = FlyteContextManager.current_context() + lt = TypeEngine.to_literal_type(typing.Dict[str, int]) + lm = TypeEngine.to_literal(ctx, {"hello": 55}, typing.Dict[str, int], lt) + lm = lm.map.to_flyte_idl() + + mock_client.get_data.return_value = dataproxy_pb2.GetDataResponse(literal_map=lm) + + rr = FlyteRemote( + Config.for_sandbox(), + default_project="flytesnacks", + default_domain="development", + ) + lr = rr.get("flyte://v1/flytesnacks/development/f6988c7bdad554a4da7a/n0/o") + assert lr.get("hello", int) == 55