Update Protos

flyteidl-protos/src/main/proto/flyteidl/admin/agent.proto

@@ -0,0 +1,88 @@
+syntax = "proto3";
+
+package flyteidl.admin;
+option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";
+
+import "flyteidl/core/literals.proto";
+import "flyteidl/core/tasks.proto";
+import "flyteidl/core/interface.proto";
+import "flyteidl/core/identifier.proto";
+
+// The state of the execution is used to control its visibility in the UI/CLI.
+enum State {
+  RETRYABLE_FAILURE = 0;
+  PERMANENT_FAILURE = 1;
+  PENDING = 2;
+  RUNNING = 3;
+  SUCCEEDED = 4;
+}
+
+// Represents a subset of runtime task execution metadata that are relevant to external plugins.
+message TaskExecutionMetadata {
+  // ID of the task execution
+  core.TaskExecutionIdentifier task_execution_id = 1;
+  // k8s namespace where the task is executed in
+  string namespace = 2;
+  // Labels attached to the task execution
+  map<string, string> labels = 3;
+  // Annotations attached to the task execution
+  map<string, string> annotations = 4;
+  // k8s service account associated with the task execution
+  string k8s_service_account = 5;
+  // Environment variables attached to the task execution
+  map<string, string> environment_variables = 6;
+}
+
+// Represents a request structure to create task.
+message CreateTaskRequest {
+  // The inputs required to start the execution. All required inputs must be
+  // included in this map. If not required and not provided, defaults apply.
+  // +optional
+  core.LiteralMap inputs = 1;
+  // Template of the task that encapsulates all the metadata of the task.
+  core.TaskTemplate template = 2;
+  // Prefix for where task output data will be written. (e.g. s3://my-bucket/randomstring)
+  string output_prefix = 3;
+  // subset of runtime task execution metadata.
+  TaskExecutionMetadata task_execution_metadata = 4;
+}
+
+// Represents a create response structure.
+message CreateTaskResponse {
+  // Metadata is created by the agent. It could be a string (jobId) or a dict (more complex metadata).
+  bytes resource_meta = 1;
+}
+
+// A message used to fetch a job resource from flyte agent server.
+message GetTaskRequest {
+  // A predefined yet extensible Task type identifier.
+  string task_type = 1;
+  // Metadata about the resource to be pass to the agent.
+  bytes resource_meta = 2;
+}
+
+// Response to get an individual task resource.
+message GetTaskResponse {
+  Resource resource = 1;
+}
+
+message Resource {
+  // The state of the execution is used to control its visibility in the UI/CLI.
+  State state = 1;
+  // The outputs of the execution. It's typically used by sql task. Agent service will create a
+  // Structured dataset pointing to the query result table.
+  // +optional
+  core.LiteralMap outputs = 2;
+}
+
+// A message used to delete a task.
+message DeleteTaskRequest {
+  // A predefined yet extensible Task type identifier.
+  string task_type = 1;
+  // Metadata about the resource to be pass to the agent.
+  bytes resource_meta = 2;
+}
+
+// Response to delete a task.
+message DeleteTaskResponse {
+}

flyteidl-protos/src/main/proto/flyteidl/admin/cluster_assignment.proto

@@ -6,45 +6,6 @@ option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";
 
 // Encapsulates specifications for routing an execution onto a specific cluster.
 message ClusterAssignment {
-  Affinity affinity = 1;
-
-  Toleration toleration = 2;
-}
-
-// Defines a set of constraints used to select eligible objects based on labels they possess.
-message Affinity {
-  // Multiples selectors are 'and'-ed together to produce the list of matching, eligible objects.
-  repeated Selector selectors = 1;
-}
-
-// Defines a set of specific label selectors that the execution can tolerate on a cluster.
-message Toleration {
-
-  // A toleration selector is similar to that of an affinity but the only valid operators are EQUALS AND EXISTS.
-  repeated Selector selectors = 1;
+  reserved 1, 2;
+  string cluster_pool_name = 3;
 }
-
-// A Selector is a specification for identifying a set of objects with corresponding labels.
-message Selector {
-
-  // The label key.
-  string key = 1;
-
-  // One or more values used to match labels.
-  // For equality (or inequality) requirements, values must contain a single element.
-  // For set-based requirements, values may contain one or more elements.
-  repeated string value = 2;
-
-  // Defines how a label with a corresponding key and value is selected or excluded.
-  enum Operator {
-    EQUALS = 0;
-    NOT_EQUALS = 1;
-    IN = 2;
-    NOT_IN = 3;
-    EXISTS = 4; // A label key with any value
-
-    // K8s supports more operators, we can consider adding them if necessary
-  }
-  Operator operator = 3;
-}
-

flyteidl-protos/src/main/proto/flyteidl/admin/common.proto

@@ -5,6 +5,8 @@ option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";
 
 import "flyteidl/core/execution.proto";
 import "flyteidl/core/identifier.proto";
+import "flyteidl/core/literals.proto";
+import "google/protobuf/timestamp.proto";
 
 // Encapsulation of fields that identifies a Flyte resource.
 // A Flyte resource can be a task, workflow or launch plan.
@@ -279,6 +281,14 @@ message Annotations {
     map<string, string> values = 1;
 }
 
+// Environment variable values to be applied to an execution resource.
+// In the future a mode (e.g. OVERRIDE, APPEND, etc) can be defined
+// to specify how to merge environment variables defined at registration and execution time.
+message Envs {
+    // Map of custom environment variables to be applied to the execution resource.
+    repeated flyteidl.core.KeyValuePair values = 1;
+}
+
 // Defines permissions associated with executions created by this launch plan spec.
 // Use either of these roles when they have permissions required by your workflow execution.
 // Deprecated.
@@ -300,3 +310,10 @@ message RawOutputDataConfig {
     // e.g. s3://bucket/key or s3://bucket/
     string output_location_prefix = 1;
 }
+
+// These URLs are returned as part of node and task execution data requests.
+message FlyteURLs {
+    string inputs      = 1;
+    string outputs     = 2;
+    string deck        = 3;
+}

flyteidl-protos/src/main/proto/flyteidl/admin/description_entity.proto

@@ -0,0 +1,95 @@
+syntax = "proto3";
+
+package flyteidl.admin;
+option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/admin";
+
+import "flyteidl/core/identifier.proto";
+import "flyteidl/admin/common.proto";
+
+// DescriptionEntity contains detailed description for the task/workflow.
+// Documentation could provide insight into the algorithms, business use case, etc.
+message DescriptionEntity {
+  // id represents the unique identifier of the description entity.
+  core.Identifier id = 1;
+  // One-liner overview of the entity.
+  string short_description = 2;
+  // Full user description with formatting preserved.
+  Description long_description = 3;
+  // Optional link to source code used to define this entity.
+  SourceCode source_code = 4;
+  // User-specified tags. These are arbitrary and can be used for searching
+  // filtering and discovering tasks.
+  repeated string tags = 5;
+}
+
+// The format of the long description
+enum DescriptionFormat {
+  DESCRIPTION_FORMAT_UNKNOWN = 0;
+  DESCRIPTION_FORMAT_MARKDOWN = 1;
+  DESCRIPTION_FORMAT_HTML = 2;
+  // python default documentation - comments is rst
+  DESCRIPTION_FORMAT_RST = 3;
+}
+
+// Full user description with formatting preserved. This can be rendered
+// by clients, such as the console or command line tools with in-tact
+// formatting.
+message Description {
+  oneof content {
+    // long description - no more than 4KB
+    string value = 1;
+    // if the description sizes exceed some threshold we can offload the entire
+    // description proto altogether to an external data store, like S3 rather than store inline in the db
+    string uri = 2;
+  }
+
+  // Format of the long description
+  DescriptionFormat format = 3;
+  // Optional link to an icon for the entity
+  string icon_link = 4;
+}
+
+// Link to source code used to define this entity
+message SourceCode {
+  string link = 1;
+}
+
+// Represents a list of DescriptionEntities returned from the admin.
+// See :ref:`ref_flyteidl.admin.DescriptionEntity` for more details
+message DescriptionEntityList {
+  // A list of DescriptionEntities returned based on the request.
+  repeated DescriptionEntity descriptionEntities = 1;
+
+  // In the case of multiple pages of results, the server-provided token can be used to fetch the next page
+  // in a query. If there are no more results, this value will be empty.
+  string token                = 2;
+}
+
+// Represents a request structure to retrieve a list of DescriptionEntities.
+// See :ref:`ref_flyteidl.admin.DescriptionEntity` for more details
+message DescriptionEntityListRequest {
+  // Identifies the specific type of resource that this identifier corresponds to.
+  flyteidl.core.ResourceType resource_type  = 1;
+
+  // The identifier for the description entity.
+  // +required
+  NamedEntityIdentifier id = 2;
+
+  // Indicates the number of resources to be returned.
+  // +required
+  uint32 limit    = 3;
+
+  // In the case of multiple pages of results, the server-provided token can be used to fetch the next page
+  // in a query.
+  // +optional
+  string token    = 4;
+
+  // Indicates a list of filters passed as string.
+  // More info on constructing filters : <Link>
+  // +optional
+  string filters  = 5;
+
+  // Sort ordering for returned list.
+  // +optional
+  Sort sort_by    = 6;
+}

flyteidl-protos/src/main/proto/flyteidl/admin/execution.proto

@@ -8,6 +8,7 @@ import "flyteidl/admin/common.proto";
 import "flyteidl/core/literals.proto";
 import "flyteidl/core/execution.proto";
 import "flyteidl/core/identifier.proto";
+import "flyteidl/core/metrics.proto";
 import "flyteidl/core/security.proto";
 import "google/protobuf/duration.proto";
 import "google/protobuf/timestamp.proto";
@@ -45,10 +46,18 @@ message ExecutionRelaunchRequest {
     // +required
     core.WorkflowExecutionIdentifier id = 1;
 
+    // Deprecated field, do not use.
+    reserved 2;
+
     // User provided value for the relaunched execution.
     // If none is provided the system will generate a unique string.
     // +optional
     string name = 3;
+
+    // Allows for all cached values of a workflow and its tasks to be overwritten for a single execution.
+    // If enabled, all calculations are performed even if cached results would be available, overwriting the stored
+    // data once execution finishes successfully.
+    bool overwrite_cache = 4;
 }
 
 // Request to recover the referenced execution.
@@ -181,6 +190,9 @@ message SystemMetadata {
 
     // Which execution cluster this execution ran on.
     string execution_cluster = 1;
+
+    // Which kubernetes namespace the execution ran under.
+    string namespace = 2;
 }
 
 // Represents attributes about an execution which are not required to launch the execution but are useful to record.
@@ -296,6 +308,16 @@ message ExecutionSpec {
     // As we need to distinguish between the field not being provided and its default value false, we have to use a wrapper
     // around the bool field.
     google.protobuf.BoolValue interruptible = 21;
+
+    // Allows for all cached values of a workflow and its tasks to be overwritten for a single execution.
+    // If enabled, all calculations are performed even if cached results would be available, overwriting the stored
+    // data once execution finishes successfully.
+    bool overwrite_cache = 22;
+
+    // Environment variables to be set for the execution.
+    Envs envs = 23;
+    // Tags to be set for the execution.
+    repeated string tags = 24;
 }
 
 // Request to terminate an in-progress execution.  This action is irreversible.
@@ -368,3 +390,19 @@ message ExecutionStateChangeDetails {
 }
 
 message ExecutionUpdateResponse {}
+
+// WorkflowExecutionGetMetricsRequest represents a request to retrieve metrics for the specified workflow execution.
+message WorkflowExecutionGetMetricsRequest {
+    // id defines the workflow execution to query for.
+    core.WorkflowExecutionIdentifier id = 1;
+
+    // depth defines the number of Flyte entity levels to traverse when breaking down execution details.
+    int32 depth = 2;
+}
+
+// WorkflowExecutionGetMetricsResponse represents the response containing metrics for the specified workflow execution.
+message WorkflowExecutionGetMetricsResponse {
+    // Span defines the top-level breakdown of the workflows execution. More precise information is nested in a
+    // hierarchical structure using Flyte entity references.
+    core.Span span = 1;
+}

flyteidl-protos/src/main/proto/flyteidl/admin/launch_plan.proto

@@ -125,6 +125,14 @@ message LaunchPlanSpec {
     // As we need to distinguish between the field not being provided and its default value false, we have to use a wrapper
     // around the bool field.
     google.protobuf.BoolValue interruptible = 19;
+
+    // Allows for all cached values of a workflow and its tasks to be overwritten for a single execution.
+    // If enabled, all calculations are performed even if cached results would be available, overwriting the stored
+    // data once execution finishes successfully.
+    bool overwrite_cache = 20;
+
+    // Environment variables to be set for the execution.
+    Envs envs = 21;
 }
 
 // Values computed by the flyte platform after launch plan registration.

flyteidl-protos/src/main/proto/flyteidl/admin/matchable_resource.proto

@@ -123,6 +123,14 @@ message WorkflowExecutionConfig {
   // As we need to distinguish between the field not being provided and its default value false, we have to use a wrapper
   // around the bool field.
   google.protobuf.BoolValue interruptible = 6;
+
+  // Allows for all cached values of a workflow and its tasks to be overwritten for a single execution.
+  // If enabled, all calculations are performed even if cached results would be available, overwriting the stored
+  // data once execution finishes successfully.
+  bool overwrite_cache = 7;
+
+  // Environment variables to be set for the execution.
+  Envs envs = 8;
 }
 
 // Generic container for encapsulating all types of the above attributes messages.