From 0a92e7e06385a34ef4cacd004fd35151400d4ea3 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 14 Dec 2024 00:33:34 +0800 Subject: [PATCH 01/13] test Signed-off-by: Nelson Chen --- flyteidl/clients/go/assets/admin.swagger.json | 4 + .../gen/pb-es/flyteidl/core/workflow_pb.ts | 10 ++- .../gen/pb-go/flyteidl/core/workflow.pb.go | 84 +++++++++++-------- .../flyteidl/service/admin.swagger.json | 4 + .../flyteidl/service/agent.swagger.json | 4 + flyteidl/gen/pb-js/flyteidl.d.ts | 6 ++ flyteidl/gen/pb-js/flyteidl.js | 19 +++++ .../pb_python/flyteidl/core/workflow_pb2.py | 8 +- .../pb_python/flyteidl/core/workflow_pb2.pyi | 6 +- flyteidl/gen/pb_rust/flyteidl.core.rs | 3 + flyteidl/protos/flyteidl/core/workflow.proto | 4 + .../pluginmachinery/core/exec_metadata.go | 1 + .../core/mocks/task_overrides.go | 34 ++++++++ .../flytek8s/config/k8spluginconfig_flags.go | 1 + .../config/k8spluginconfig_flags_test.go | 14 ++++ .../flytek8s/plugin_exec_context.go | 4 + flyteplugins/tests/end_to_end.go | 1 + .../pkg/apis/flyteworkflow/v1alpha1/iface.go | 1 + .../pkg/apis/flyteworkflow/v1alpha1/nodes.go | 6 ++ .../pkg/compiler/transformers/k8s/node.go | 6 ++ 20 files changed, 178 insertions(+), 42 deletions(-) diff --git a/flyteidl/clients/go/assets/admin.swagger.json b/flyteidl/clients/go/assets/admin.swagger.json index 01ae020a09..8c450b3ac4 100644 --- a/flyteidl/clients/go/assets/admin.swagger.json +++ b/flyteidl/clients/go/assets/admin.swagger.json @@ -8313,6 +8313,10 @@ "container_image": { "type": "string", "description": "Override for the image used by task pods." + }, + "pod_template": { + "$ref": "#/definitions/coreK8sPod", + "title": "Override for the pod template used by task pods" } }, "description": "Optional task node overrides that will be applied at task execution time." diff --git a/flyteidl/gen/pb-es/flyteidl/core/workflow_pb.ts b/flyteidl/gen/pb-es/flyteidl/core/workflow_pb.ts index 7dce890e0d..05f2d62dd2 100644 --- a/flyteidl/gen/pb-es/flyteidl/core/workflow_pb.ts +++ b/flyteidl/gen/pb-es/flyteidl/core/workflow_pb.ts @@ -11,7 +11,7 @@ import { Identifier } from "./identifier_pb.js"; import { Binding, LiteralMap, RetryStrategy } from "./literals_pb.js"; import { QualityOfService } from "./execution_pb.js"; import { TypedInterface } from "./interface_pb.js"; -import { ExtendedResources, Resources } from "./tasks_pb.js"; +import { ExtendedResources, K8sPod, Resources } from "./tasks_pb.js"; /** * Defines a condition and the execution unit that should be executed if the condition is satisfied. @@ -1176,6 +1176,13 @@ export class TaskNodeOverrides extends Message { */ containerImage = ""; + /** + * Override for the pod template used by task pods + * + * @generated from field: flyteidl.core.K8sPod pod_template = 4; + */ + podTemplate?: K8sPod; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -1187,6 +1194,7 @@ export class TaskNodeOverrides extends Message { { no: 1, name: "resources", kind: "message", T: Resources }, { no: 2, name: "extended_resources", kind: "message", T: ExtendedResources }, { no: 3, name: "container_image", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 4, name: "pod_template", kind: "message", T: K8sPod }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): TaskNodeOverrides { diff --git a/flyteidl/gen/pb-go/flyteidl/core/workflow.pb.go b/flyteidl/gen/pb-go/flyteidl/core/workflow.pb.go index d484bd7ae9..36b8925d24 100644 --- a/flyteidl/gen/pb-go/flyteidl/core/workflow.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/core/workflow.pb.go @@ -1589,6 +1589,8 @@ type TaskNodeOverrides struct { ExtendedResources *ExtendedResources `protobuf:"bytes,2,opt,name=extended_resources,json=extendedResources,proto3" json:"extended_resources,omitempty"` // Override for the image used by task pods. ContainerImage string `protobuf:"bytes,3,opt,name=container_image,json=containerImage,proto3" json:"container_image,omitempty"` + // Override for the pod template used by task pods + PodTemplate *K8SPod `protobuf:"bytes,4,opt,name=pod_template,json=podTemplate,proto3" json:"pod_template,omitempty"` } func (x *TaskNodeOverrides) Reset() { @@ -1644,6 +1646,13 @@ func (x *TaskNodeOverrides) GetContainerImage() string { return "" } +func (x *TaskNodeOverrides) GetPodTemplate() *K8SPod { + if x != nil { + return x.PodTemplate + } + return nil +} + // A structure that uniquely identifies a launch plan in the system. type LaunchPlanTemplate struct { state protoimpl.MessageState @@ -1955,7 +1964,7 @@ var file_flyteidl_core_workflow_proto_rawDesc = []byte{ 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x52, 0x10, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, - 0x22, 0xc5, 0x01, 0x0a, 0x11, 0x54, 0x61, 0x73, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x4f, 0x76, 0x65, + 0x22, 0xff, 0x01, 0x0a, 0x11, 0x54, 0x61, 0x73, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x12, 0x36, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, @@ -1967,31 +1976,34 @@ var file_flyteidl_core_workflow_proto_rawDesc = []byte{ 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x22, 0xba, 0x01, 0x0a, 0x12, 0x4c, 0x61, 0x75, - 0x6e, 0x63, 0x68, 0x50, 0x6c, 0x61, 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, - 0x29, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, - 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x49, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x52, 0x02, 0x69, 0x64, 0x12, 0x3b, 0x0a, 0x09, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, - 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x54, 0x79, - 0x70, 0x65, 0x64, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x52, 0x09, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x3c, 0x0a, 0x0c, 0x66, 0x69, 0x78, 0x65, 0x64, - 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, - 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, - 0x74, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x61, 0x70, 0x52, 0x0b, 0x66, 0x69, 0x78, 0x65, 0x64, 0x49, - 0x6e, 0x70, 0x75, 0x74, 0x73, 0x42, 0xb3, 0x01, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x2e, 0x66, 0x6c, - 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0d, 0x57, 0x6f, 0x72, - 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3a, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x6f, 0x72, - 0x67, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, - 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x62, 0x2d, 0x67, 0x6f, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, - 0x69, 0x64, 0x6c, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0xa2, 0x02, 0x03, 0x46, 0x43, 0x58, 0xaa, 0x02, - 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0xca, 0x02, - 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0xe2, 0x02, - 0x19, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0x5c, 0x47, - 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0e, 0x46, 0x6c, 0x79, - 0x74, 0x65, 0x69, 0x64, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x6e, 0x65, 0x72, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x38, 0x0a, 0x0c, 0x70, 0x6f, 0x64, 0x5f, + 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, + 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4b, + 0x38, 0x73, 0x50, 0x6f, 0x64, 0x52, 0x0b, 0x70, 0x6f, 0x64, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x22, 0xba, 0x01, 0x0a, 0x12, 0x4c, 0x61, 0x75, 0x6e, 0x63, 0x68, 0x50, 0x6c, 0x61, + 0x6e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x29, 0x0a, 0x02, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, + 0x52, 0x02, 0x69, 0x64, 0x12, 0x3b, 0x0a, 0x09, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, + 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x64, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x52, 0x09, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, + 0x65, 0x12, 0x3c, 0x0a, 0x0c, 0x66, 0x69, 0x78, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, + 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x4d, + 0x61, 0x70, 0x52, 0x0b, 0x66, 0x69, 0x78, 0x65, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x42, + 0xb3, 0x01, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0d, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x6f, 0x72, 0x67, 0x2f, 0x66, 0x6c, 0x79, 0x74, + 0x65, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, + 0x62, 0x2d, 0x67, 0x6f, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x63, 0x6f, + 0x72, 0x65, 0xa2, 0x02, 0x03, 0x46, 0x43, 0x58, 0xaa, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, + 0x69, 0x64, 0x6c, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0xca, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, + 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0xe2, 0x02, 0x19, 0x46, 0x6c, 0x79, 0x74, 0x65, + 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0e, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x3a, + 0x3a, 0x43, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2042,7 +2054,8 @@ var file_flyteidl_core_workflow_proto_goTypes = []interface{}{ (*TypedInterface)(nil), // 30: flyteidl.core.TypedInterface (*Resources)(nil), // 31: flyteidl.core.Resources (*ExtendedResources)(nil), // 32: flyteidl.core.ExtendedResources - (*LiteralMap)(nil), // 33: flyteidl.core.LiteralMap + (*K8SPod)(nil), // 33: flyteidl.core.K8sPod + (*LiteralMap)(nil), // 34: flyteidl.core.LiteralMap } var file_flyteidl_core_workflow_proto_depIdxs = []int32{ 21, // 0: flyteidl.core.IfBlock.condition:type_name -> flyteidl.core.BooleanExpression @@ -2086,14 +2099,15 @@ var file_flyteidl_core_workflow_proto_depIdxs = []int32{ 16, // 38: flyteidl.core.WorkflowTemplate.metadata_defaults:type_name -> flyteidl.core.WorkflowMetadataDefaults 31, // 39: flyteidl.core.TaskNodeOverrides.resources:type_name -> flyteidl.core.Resources 32, // 40: flyteidl.core.TaskNodeOverrides.extended_resources:type_name -> flyteidl.core.ExtendedResources - 23, // 41: flyteidl.core.LaunchPlanTemplate.id:type_name -> flyteidl.core.Identifier - 30, // 42: flyteidl.core.LaunchPlanTemplate.interface:type_name -> flyteidl.core.TypedInterface - 33, // 43: flyteidl.core.LaunchPlanTemplate.fixed_inputs:type_name -> flyteidl.core.LiteralMap - 44, // [44:44] is the sub-list for method output_type - 44, // [44:44] is the sub-list for method input_type - 44, // [44:44] is the sub-list for extension type_name - 44, // [44:44] is the sub-list for extension extendee - 0, // [0:44] is the sub-list for field type_name + 33, // 41: flyteidl.core.TaskNodeOverrides.pod_template:type_name -> flyteidl.core.K8sPod + 23, // 42: flyteidl.core.LaunchPlanTemplate.id:type_name -> flyteidl.core.Identifier + 30, // 43: flyteidl.core.LaunchPlanTemplate.interface:type_name -> flyteidl.core.TypedInterface + 34, // 44: flyteidl.core.LaunchPlanTemplate.fixed_inputs:type_name -> flyteidl.core.LiteralMap + 45, // [45:45] is the sub-list for method output_type + 45, // [45:45] is the sub-list for method input_type + 45, // [45:45] is the sub-list for extension type_name + 45, // [45:45] is the sub-list for extension extendee + 0, // [0:45] is the sub-list for field type_name } func init() { file_flyteidl_core_workflow_proto_init() } diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json index 01ae020a09..8c450b3ac4 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json @@ -8313,6 +8313,10 @@ "container_image": { "type": "string", "description": "Override for the image used by task pods." + }, + "pod_template": { + "$ref": "#/definitions/coreK8sPod", + "title": "Override for the pod template used by task pods" } }, "description": "Optional task node overrides that will be applied at task execution time." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json index 4fbea61e15..eae5bd2df7 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json @@ -1877,6 +1877,10 @@ "container_image": { "type": "string", "description": "Override for the image used by task pods." + }, + "pod_template": { + "$ref": "#/definitions/coreK8sPod", + "title": "Override for the pod template used by task pods" } }, "description": "Optional task node overrides that will be applied at task execution time." diff --git a/flyteidl/gen/pb-js/flyteidl.d.ts b/flyteidl/gen/pb-js/flyteidl.d.ts index 09ecc02635..d80e01ddfe 100644 --- a/flyteidl/gen/pb-js/flyteidl.d.ts +++ b/flyteidl/gen/pb-js/flyteidl.d.ts @@ -5157,6 +5157,9 @@ export namespace flyteidl { /** TaskNodeOverrides containerImage */ containerImage?: (string|null); + + /** TaskNodeOverrides podTemplate */ + podTemplate?: (flyteidl.core.IK8sPod|null); } /** Represents a TaskNodeOverrides. */ @@ -5177,6 +5180,9 @@ export namespace flyteidl { /** TaskNodeOverrides containerImage. */ public containerImage: string; + /** TaskNodeOverrides podTemplate. */ + public podTemplate?: (flyteidl.core.IK8sPod|null); + /** * Creates a new TaskNodeOverrides instance using the specified properties. * @param [properties] Properties to set diff --git a/flyteidl/gen/pb-js/flyteidl.js b/flyteidl/gen/pb-js/flyteidl.js index d080bc7c85..a8722e400c 100644 --- a/flyteidl/gen/pb-js/flyteidl.js +++ b/flyteidl/gen/pb-js/flyteidl.js @@ -12501,6 +12501,7 @@ * @property {flyteidl.core.IResources|null} [resources] TaskNodeOverrides resources * @property {flyteidl.core.IExtendedResources|null} [extendedResources] TaskNodeOverrides extendedResources * @property {string|null} [containerImage] TaskNodeOverrides containerImage + * @property {flyteidl.core.IK8sPod|null} [podTemplate] TaskNodeOverrides podTemplate */ /** @@ -12542,6 +12543,14 @@ */ TaskNodeOverrides.prototype.containerImage = ""; + /** + * TaskNodeOverrides podTemplate. + * @member {flyteidl.core.IK8sPod|null|undefined} podTemplate + * @memberof flyteidl.core.TaskNodeOverrides + * @instance + */ + TaskNodeOverrides.prototype.podTemplate = null; + /** * Creates a new TaskNodeOverrides instance using the specified properties. * @function create @@ -12572,6 +12581,8 @@ $root.flyteidl.core.ExtendedResources.encode(message.extendedResources, writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim(); if (message.containerImage != null && message.hasOwnProperty("containerImage")) writer.uint32(/* id 3, wireType 2 =*/26).string(message.containerImage); + if (message.podTemplate != null && message.hasOwnProperty("podTemplate")) + $root.flyteidl.core.K8sPod.encode(message.podTemplate, writer.uint32(/* id 4, wireType 2 =*/34).fork()).ldelim(); return writer; }; @@ -12602,6 +12613,9 @@ case 3: message.containerImage = reader.string(); break; + case 4: + message.podTemplate = $root.flyteidl.core.K8sPod.decode(reader, reader.uint32()); + break; default: reader.skipType(tag & 7); break; @@ -12634,6 +12648,11 @@ if (message.containerImage != null && message.hasOwnProperty("containerImage")) if (!$util.isString(message.containerImage)) return "containerImage: string expected"; + if (message.podTemplate != null && message.hasOwnProperty("podTemplate")) { + var error = $root.flyteidl.core.K8sPod.verify(message.podTemplate); + if (error) + return "podTemplate." + error; + } return null; }; diff --git a/flyteidl/gen/pb_python/flyteidl/core/workflow_pb2.py b/flyteidl/gen/pb_python/flyteidl/core/workflow_pb2.py index 9f5b5f0881..cf41aea1c7 100644 --- a/flyteidl/gen/pb_python/flyteidl/core/workflow_pb2.py +++ b/flyteidl/gen/pb_python/flyteidl/core/workflow_pb2.py @@ -23,7 +23,7 @@ from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66lyteidl/core/workflow.proto\x12\rflyteidl.core\x1a\x1d\x66lyteidl/core/condition.proto\x1a\x1d\x66lyteidl/core/execution.proto\x1a\x1e\x66lyteidl/core/identifier.proto\x1a\x1d\x66lyteidl/core/interface.proto\x1a\x1c\x66lyteidl/core/literals.proto\x1a\x19\x66lyteidl/core/tasks.proto\x1a\x19\x66lyteidl/core/types.proto\x1a\x1c\x66lyteidl/core/security.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1egoogle/protobuf/wrappers.proto\"{\n\x07IfBlock\x12>\n\tcondition\x18\x01 \x01(\x0b\x32 .flyteidl.core.BooleanExpressionR\tcondition\x12\x30\n\tthen_node\x18\x02 \x01(\x0b\x32\x13.flyteidl.core.NodeR\x08thenNode\"\xd4\x01\n\x0bIfElseBlock\x12*\n\x04\x63\x61se\x18\x01 \x01(\x0b\x32\x16.flyteidl.core.IfBlockR\x04\x63\x61se\x12,\n\x05other\x18\x02 \x03(\x0b\x32\x16.flyteidl.core.IfBlockR\x05other\x12\x32\n\telse_node\x18\x03 \x01(\x0b\x32\x13.flyteidl.core.NodeH\x00R\x08\x65lseNode\x12,\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x14.flyteidl.core.ErrorH\x00R\x05\x65rrorB\t\n\x07\x64\x65\x66\x61ult\"A\n\nBranchNode\x12\x33\n\x07if_else\x18\x01 \x01(\x0b\x32\x1a.flyteidl.core.IfElseBlockR\x06ifElse\"\x97\x01\n\x08TaskNode\x12>\n\x0creference_id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierH\x00R\x0breferenceId\x12>\n\toverrides\x18\x02 \x01(\x0b\x32 .flyteidl.core.TaskNodeOverridesR\toverridesB\x0b\n\treference\"\xa6\x01\n\x0cWorkflowNode\x12\x42\n\x0elaunchplan_ref\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierH\x00R\rlaunchplanRef\x12\x45\n\x10sub_workflow_ref\x18\x02 \x01(\x0b\x32\x19.flyteidl.core.IdentifierH\x00R\x0esubWorkflowRefB\x0b\n\treference\"/\n\x10\x41pproveCondition\x12\x1b\n\tsignal_id\x18\x01 \x01(\tR\x08signalId\"\x90\x01\n\x0fSignalCondition\x12\x1b\n\tsignal_id\x18\x01 \x01(\tR\x08signalId\x12.\n\x04type\x18\x02 \x01(\x0b\x32\x1a.flyteidl.core.LiteralTypeR\x04type\x12\x30\n\x14output_variable_name\x18\x03 \x01(\tR\x12outputVariableName\"G\n\x0eSleepCondition\x12\x35\n\x08\x64uration\x18\x01 \x01(\x0b\x32\x19.google.protobuf.DurationR\x08\x64uration\"\xc5\x01\n\x08GateNode\x12;\n\x07\x61pprove\x18\x01 \x01(\x0b\x32\x1f.flyteidl.core.ApproveConditionH\x00R\x07\x61pprove\x12\x38\n\x06signal\x18\x02 \x01(\x0b\x32\x1e.flyteidl.core.SignalConditionH\x00R\x06signal\x12\x35\n\x05sleep\x18\x03 \x01(\x0b\x32\x1d.flyteidl.core.SleepConditionH\x00R\x05sleepB\x0b\n\tcondition\"\xba\x03\n\tArrayNode\x12\'\n\x04node\x18\x01 \x01(\x0b\x32\x13.flyteidl.core.NodeR\x04node\x12\"\n\x0bparallelism\x18\x02 \x01(\rH\x00R\x0bparallelism\x12%\n\rmin_successes\x18\x03 \x01(\rH\x01R\x0cminSuccesses\x12,\n\x11min_success_ratio\x18\x04 \x01(\x02H\x01R\x0fminSuccessRatio\x12M\n\x0e\x65xecution_mode\x18\x05 \x01(\x0e\x32&.flyteidl.core.ArrayNode.ExecutionModeR\rexecutionMode\x12^\n\x1eis_original_sub_node_interface\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.BoolValueR\x1aisOriginalSubNodeInterface\"2\n\rExecutionMode\x12\x11\n\rMINIMAL_STATE\x10\x00\x12\x0e\n\nFULL_STATE\x10\x01\x42\x14\n\x12parallelism_optionB\x12\n\x10success_criteria\"\x8c\x03\n\x0cNodeMetadata\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x33\n\x07timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\x07timeout\x12\x36\n\x07retries\x18\x05 \x01(\x0b\x32\x1c.flyteidl.core.RetryStrategyR\x07retries\x12&\n\rinterruptible\x18\x06 \x01(\x08H\x00R\rinterruptible\x12\x1e\n\tcacheable\x18\x07 \x01(\x08H\x01R\tcacheable\x12%\n\rcache_version\x18\x08 \x01(\tH\x02R\x0c\x63\x61\x63heVersion\x12/\n\x12\x63\x61\x63he_serializable\x18\t \x01(\x08H\x03R\x11\x63\x61\x63heSerializableB\x15\n\x13interruptible_valueB\x11\n\x0f\x63\x61\x63heable_valueB\x15\n\x13\x63\x61\x63he_version_valueB\x1a\n\x18\x63\x61\x63he_serializable_value\"/\n\x05\x41lias\x12\x10\n\x03var\x18\x01 \x01(\tR\x03var\x12\x14\n\x05\x61lias\x18\x02 \x01(\tR\x05\x61lias\"\x9f\x04\n\x04Node\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x37\n\x08metadata\x18\x02 \x01(\x0b\x32\x1b.flyteidl.core.NodeMetadataR\x08metadata\x12.\n\x06inputs\x18\x03 \x03(\x0b\x32\x16.flyteidl.core.BindingR\x06inputs\x12*\n\x11upstream_node_ids\x18\x04 \x03(\tR\x0fupstreamNodeIds\x12;\n\x0eoutput_aliases\x18\x05 \x03(\x0b\x32\x14.flyteidl.core.AliasR\routputAliases\x12\x36\n\ttask_node\x18\x06 \x01(\x0b\x32\x17.flyteidl.core.TaskNodeH\x00R\x08taskNode\x12\x42\n\rworkflow_node\x18\x07 \x01(\x0b\x32\x1b.flyteidl.core.WorkflowNodeH\x00R\x0cworkflowNode\x12<\n\x0b\x62ranch_node\x18\x08 \x01(\x0b\x32\x19.flyteidl.core.BranchNodeH\x00R\nbranchNode\x12\x36\n\tgate_node\x18\t \x01(\x0b\x32\x17.flyteidl.core.GateNodeH\x00R\x08gateNode\x12\x39\n\narray_node\x18\n \x01(\x0b\x32\x18.flyteidl.core.ArrayNodeH\x00R\tarrayNodeB\x08\n\x06target\"\xfc\x02\n\x10WorkflowMetadata\x12M\n\x12quality_of_service\x18\x01 \x01(\x0b\x32\x1f.flyteidl.core.QualityOfServiceR\x10qualityOfService\x12N\n\non_failure\x18\x02 \x01(\x0e\x32/.flyteidl.core.WorkflowMetadata.OnFailurePolicyR\tonFailure\x12=\n\x04tags\x18\x03 \x03(\x0b\x32).flyteidl.core.WorkflowMetadata.TagsEntryR\x04tags\x1a\x37\n\tTagsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"Q\n\x0fOnFailurePolicy\x12\x14\n\x10\x46\x41IL_IMMEDIATELY\x10\x00\x12(\n$FAIL_AFTER_EXECUTABLE_NODES_COMPLETE\x10\x01\"@\n\x18WorkflowMetadataDefaults\x12$\n\rinterruptible\x18\x01 \x01(\x08R\rinterruptible\"\xa2\x03\n\x10WorkflowTemplate\x12)\n\x02id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierR\x02id\x12;\n\x08metadata\x18\x02 \x01(\x0b\x32\x1f.flyteidl.core.WorkflowMetadataR\x08metadata\x12;\n\tinterface\x18\x03 \x01(\x0b\x32\x1d.flyteidl.core.TypedInterfaceR\tinterface\x12)\n\x05nodes\x18\x04 \x03(\x0b\x32\x13.flyteidl.core.NodeR\x05nodes\x12\x30\n\x07outputs\x18\x05 \x03(\x0b\x32\x16.flyteidl.core.BindingR\x07outputs\x12\x36\n\x0c\x66\x61ilure_node\x18\x06 \x01(\x0b\x32\x13.flyteidl.core.NodeR\x0b\x66\x61ilureNode\x12T\n\x11metadata_defaults\x18\x07 \x01(\x0b\x32\'.flyteidl.core.WorkflowMetadataDefaultsR\x10metadataDefaults\"\xc5\x01\n\x11TaskNodeOverrides\x12\x36\n\tresources\x18\x01 \x01(\x0b\x32\x18.flyteidl.core.ResourcesR\tresources\x12O\n\x12\x65xtended_resources\x18\x02 \x01(\x0b\x32 .flyteidl.core.ExtendedResourcesR\x11\x65xtendedResources\x12\'\n\x0f\x63ontainer_image\x18\x03 \x01(\tR\x0e\x63ontainerImage\"\xba\x01\n\x12LaunchPlanTemplate\x12)\n\x02id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierR\x02id\x12;\n\tinterface\x18\x02 \x01(\x0b\x32\x1d.flyteidl.core.TypedInterfaceR\tinterface\x12<\n\x0c\x66ixed_inputs\x18\x03 \x01(\x0b\x32\x19.flyteidl.core.LiteralMapR\x0b\x66ixedInputsB\xb3\x01\n\x11\x63om.flyteidl.coreB\rWorkflowProtoP\x01Z:github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core\xa2\x02\x03\x46\x43X\xaa\x02\rFlyteidl.Core\xca\x02\rFlyteidl\\Core\xe2\x02\x19\x46lyteidl\\Core\\GPBMetadata\xea\x02\x0e\x46lyteidl::Coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66lyteidl/core/workflow.proto\x12\rflyteidl.core\x1a\x1d\x66lyteidl/core/condition.proto\x1a\x1d\x66lyteidl/core/execution.proto\x1a\x1e\x66lyteidl/core/identifier.proto\x1a\x1d\x66lyteidl/core/interface.proto\x1a\x1c\x66lyteidl/core/literals.proto\x1a\x19\x66lyteidl/core/tasks.proto\x1a\x19\x66lyteidl/core/types.proto\x1a\x1c\x66lyteidl/core/security.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1egoogle/protobuf/wrappers.proto\"{\n\x07IfBlock\x12>\n\tcondition\x18\x01 \x01(\x0b\x32 .flyteidl.core.BooleanExpressionR\tcondition\x12\x30\n\tthen_node\x18\x02 \x01(\x0b\x32\x13.flyteidl.core.NodeR\x08thenNode\"\xd4\x01\n\x0bIfElseBlock\x12*\n\x04\x63\x61se\x18\x01 \x01(\x0b\x32\x16.flyteidl.core.IfBlockR\x04\x63\x61se\x12,\n\x05other\x18\x02 \x03(\x0b\x32\x16.flyteidl.core.IfBlockR\x05other\x12\x32\n\telse_node\x18\x03 \x01(\x0b\x32\x13.flyteidl.core.NodeH\x00R\x08\x65lseNode\x12,\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x14.flyteidl.core.ErrorH\x00R\x05\x65rrorB\t\n\x07\x64\x65\x66\x61ult\"A\n\nBranchNode\x12\x33\n\x07if_else\x18\x01 \x01(\x0b\x32\x1a.flyteidl.core.IfElseBlockR\x06ifElse\"\x97\x01\n\x08TaskNode\x12>\n\x0creference_id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierH\x00R\x0breferenceId\x12>\n\toverrides\x18\x02 \x01(\x0b\x32 .flyteidl.core.TaskNodeOverridesR\toverridesB\x0b\n\treference\"\xa6\x01\n\x0cWorkflowNode\x12\x42\n\x0elaunchplan_ref\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierH\x00R\rlaunchplanRef\x12\x45\n\x10sub_workflow_ref\x18\x02 \x01(\x0b\x32\x19.flyteidl.core.IdentifierH\x00R\x0esubWorkflowRefB\x0b\n\treference\"/\n\x10\x41pproveCondition\x12\x1b\n\tsignal_id\x18\x01 \x01(\tR\x08signalId\"\x90\x01\n\x0fSignalCondition\x12\x1b\n\tsignal_id\x18\x01 \x01(\tR\x08signalId\x12.\n\x04type\x18\x02 \x01(\x0b\x32\x1a.flyteidl.core.LiteralTypeR\x04type\x12\x30\n\x14output_variable_name\x18\x03 \x01(\tR\x12outputVariableName\"G\n\x0eSleepCondition\x12\x35\n\x08\x64uration\x18\x01 \x01(\x0b\x32\x19.google.protobuf.DurationR\x08\x64uration\"\xc5\x01\n\x08GateNode\x12;\n\x07\x61pprove\x18\x01 \x01(\x0b\x32\x1f.flyteidl.core.ApproveConditionH\x00R\x07\x61pprove\x12\x38\n\x06signal\x18\x02 \x01(\x0b\x32\x1e.flyteidl.core.SignalConditionH\x00R\x06signal\x12\x35\n\x05sleep\x18\x03 \x01(\x0b\x32\x1d.flyteidl.core.SleepConditionH\x00R\x05sleepB\x0b\n\tcondition\"\xba\x03\n\tArrayNode\x12\'\n\x04node\x18\x01 \x01(\x0b\x32\x13.flyteidl.core.NodeR\x04node\x12\"\n\x0bparallelism\x18\x02 \x01(\rH\x00R\x0bparallelism\x12%\n\rmin_successes\x18\x03 \x01(\rH\x01R\x0cminSuccesses\x12,\n\x11min_success_ratio\x18\x04 \x01(\x02H\x01R\x0fminSuccessRatio\x12M\n\x0e\x65xecution_mode\x18\x05 \x01(\x0e\x32&.flyteidl.core.ArrayNode.ExecutionModeR\rexecutionMode\x12^\n\x1eis_original_sub_node_interface\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.BoolValueR\x1aisOriginalSubNodeInterface\"2\n\rExecutionMode\x12\x11\n\rMINIMAL_STATE\x10\x00\x12\x0e\n\nFULL_STATE\x10\x01\x42\x14\n\x12parallelism_optionB\x12\n\x10success_criteria\"\x8c\x03\n\x0cNodeMetadata\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x33\n\x07timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\x07timeout\x12\x36\n\x07retries\x18\x05 \x01(\x0b\x32\x1c.flyteidl.core.RetryStrategyR\x07retries\x12&\n\rinterruptible\x18\x06 \x01(\x08H\x00R\rinterruptible\x12\x1e\n\tcacheable\x18\x07 \x01(\x08H\x01R\tcacheable\x12%\n\rcache_version\x18\x08 \x01(\tH\x02R\x0c\x63\x61\x63heVersion\x12/\n\x12\x63\x61\x63he_serializable\x18\t \x01(\x08H\x03R\x11\x63\x61\x63heSerializableB\x15\n\x13interruptible_valueB\x11\n\x0f\x63\x61\x63heable_valueB\x15\n\x13\x63\x61\x63he_version_valueB\x1a\n\x18\x63\x61\x63he_serializable_value\"/\n\x05\x41lias\x12\x10\n\x03var\x18\x01 \x01(\tR\x03var\x12\x14\n\x05\x61lias\x18\x02 \x01(\tR\x05\x61lias\"\x9f\x04\n\x04Node\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x37\n\x08metadata\x18\x02 \x01(\x0b\x32\x1b.flyteidl.core.NodeMetadataR\x08metadata\x12.\n\x06inputs\x18\x03 \x03(\x0b\x32\x16.flyteidl.core.BindingR\x06inputs\x12*\n\x11upstream_node_ids\x18\x04 \x03(\tR\x0fupstreamNodeIds\x12;\n\x0eoutput_aliases\x18\x05 \x03(\x0b\x32\x14.flyteidl.core.AliasR\routputAliases\x12\x36\n\ttask_node\x18\x06 \x01(\x0b\x32\x17.flyteidl.core.TaskNodeH\x00R\x08taskNode\x12\x42\n\rworkflow_node\x18\x07 \x01(\x0b\x32\x1b.flyteidl.core.WorkflowNodeH\x00R\x0cworkflowNode\x12<\n\x0b\x62ranch_node\x18\x08 \x01(\x0b\x32\x19.flyteidl.core.BranchNodeH\x00R\nbranchNode\x12\x36\n\tgate_node\x18\t \x01(\x0b\x32\x17.flyteidl.core.GateNodeH\x00R\x08gateNode\x12\x39\n\narray_node\x18\n \x01(\x0b\x32\x18.flyteidl.core.ArrayNodeH\x00R\tarrayNodeB\x08\n\x06target\"\xfc\x02\n\x10WorkflowMetadata\x12M\n\x12quality_of_service\x18\x01 \x01(\x0b\x32\x1f.flyteidl.core.QualityOfServiceR\x10qualityOfService\x12N\n\non_failure\x18\x02 \x01(\x0e\x32/.flyteidl.core.WorkflowMetadata.OnFailurePolicyR\tonFailure\x12=\n\x04tags\x18\x03 \x03(\x0b\x32).flyteidl.core.WorkflowMetadata.TagsEntryR\x04tags\x1a\x37\n\tTagsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"Q\n\x0fOnFailurePolicy\x12\x14\n\x10\x46\x41IL_IMMEDIATELY\x10\x00\x12(\n$FAIL_AFTER_EXECUTABLE_NODES_COMPLETE\x10\x01\"@\n\x18WorkflowMetadataDefaults\x12$\n\rinterruptible\x18\x01 \x01(\x08R\rinterruptible\"\xa2\x03\n\x10WorkflowTemplate\x12)\n\x02id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierR\x02id\x12;\n\x08metadata\x18\x02 \x01(\x0b\x32\x1f.flyteidl.core.WorkflowMetadataR\x08metadata\x12;\n\tinterface\x18\x03 \x01(\x0b\x32\x1d.flyteidl.core.TypedInterfaceR\tinterface\x12)\n\x05nodes\x18\x04 \x03(\x0b\x32\x13.flyteidl.core.NodeR\x05nodes\x12\x30\n\x07outputs\x18\x05 \x03(\x0b\x32\x16.flyteidl.core.BindingR\x07outputs\x12\x36\n\x0c\x66\x61ilure_node\x18\x06 \x01(\x0b\x32\x13.flyteidl.core.NodeR\x0b\x66\x61ilureNode\x12T\n\x11metadata_defaults\x18\x07 \x01(\x0b\x32\'.flyteidl.core.WorkflowMetadataDefaultsR\x10metadataDefaults\"\xff\x01\n\x11TaskNodeOverrides\x12\x36\n\tresources\x18\x01 \x01(\x0b\x32\x18.flyteidl.core.ResourcesR\tresources\x12O\n\x12\x65xtended_resources\x18\x02 \x01(\x0b\x32 .flyteidl.core.ExtendedResourcesR\x11\x65xtendedResources\x12\'\n\x0f\x63ontainer_image\x18\x03 \x01(\tR\x0e\x63ontainerImage\x12\x38\n\x0cpod_template\x18\x04 \x01(\x0b\x32\x15.flyteidl.core.K8sPodR\x0bpodTemplate\"\xba\x01\n\x12LaunchPlanTemplate\x12)\n\x02id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierR\x02id\x12;\n\tinterface\x18\x02 \x01(\x0b\x32\x1d.flyteidl.core.TypedInterfaceR\tinterface\x12<\n\x0c\x66ixed_inputs\x18\x03 \x01(\x0b\x32\x19.flyteidl.core.LiteralMapR\x0b\x66ixedInputsB\xb3\x01\n\x11\x63om.flyteidl.coreB\rWorkflowProtoP\x01Z:github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core\xa2\x02\x03\x46\x43X\xaa\x02\rFlyteidl.Core\xca\x02\rFlyteidl\\Core\xe2\x02\x19\x46lyteidl\\Core\\GPBMetadata\xea\x02\x0e\x46lyteidl::Coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -73,7 +73,7 @@ _globals['_WORKFLOWTEMPLATE']._serialized_start=3438 _globals['_WORKFLOWTEMPLATE']._serialized_end=3856 _globals['_TASKNODEOVERRIDES']._serialized_start=3859 - _globals['_TASKNODEOVERRIDES']._serialized_end=4056 - _globals['_LAUNCHPLANTEMPLATE']._serialized_start=4059 - _globals['_LAUNCHPLANTEMPLATE']._serialized_end=4245 + _globals['_TASKNODEOVERRIDES']._serialized_end=4114 + _globals['_LAUNCHPLANTEMPLATE']._serialized_start=4117 + _globals['_LAUNCHPLANTEMPLATE']._serialized_end=4303 # @@protoc_insertion_point(module_scope) diff --git a/flyteidl/gen/pb_python/flyteidl/core/workflow_pb2.pyi b/flyteidl/gen/pb_python/flyteidl/core/workflow_pb2.pyi index 6099ea6d6e..aaee006e2e 100644 --- a/flyteidl/gen/pb_python/flyteidl/core/workflow_pb2.pyi +++ b/flyteidl/gen/pb_python/flyteidl/core/workflow_pb2.pyi @@ -210,14 +210,16 @@ class WorkflowTemplate(_message.Message): def __init__(self, id: _Optional[_Union[_identifier_pb2.Identifier, _Mapping]] = ..., metadata: _Optional[_Union[WorkflowMetadata, _Mapping]] = ..., interface: _Optional[_Union[_interface_pb2.TypedInterface, _Mapping]] = ..., nodes: _Optional[_Iterable[_Union[Node, _Mapping]]] = ..., outputs: _Optional[_Iterable[_Union[_literals_pb2.Binding, _Mapping]]] = ..., failure_node: _Optional[_Union[Node, _Mapping]] = ..., metadata_defaults: _Optional[_Union[WorkflowMetadataDefaults, _Mapping]] = ...) -> None: ... class TaskNodeOverrides(_message.Message): - __slots__ = ["resources", "extended_resources", "container_image"] + __slots__ = ["resources", "extended_resources", "container_image", "pod_template"] RESOURCES_FIELD_NUMBER: _ClassVar[int] EXTENDED_RESOURCES_FIELD_NUMBER: _ClassVar[int] CONTAINER_IMAGE_FIELD_NUMBER: _ClassVar[int] + POD_TEMPLATE_FIELD_NUMBER: _ClassVar[int] resources: _tasks_pb2.Resources extended_resources: _tasks_pb2.ExtendedResources container_image: str - def __init__(self, resources: _Optional[_Union[_tasks_pb2.Resources, _Mapping]] = ..., extended_resources: _Optional[_Union[_tasks_pb2.ExtendedResources, _Mapping]] = ..., container_image: _Optional[str] = ...) -> None: ... + pod_template: _tasks_pb2.K8sPod + def __init__(self, resources: _Optional[_Union[_tasks_pb2.Resources, _Mapping]] = ..., extended_resources: _Optional[_Union[_tasks_pb2.ExtendedResources, _Mapping]] = ..., container_image: _Optional[str] = ..., pod_template: _Optional[_Union[_tasks_pb2.K8sPod, _Mapping]] = ...) -> None: ... class LaunchPlanTemplate(_message.Message): __slots__ = ["id", "interface", "fixed_inputs"] diff --git a/flyteidl/gen/pb_rust/flyteidl.core.rs b/flyteidl/gen/pb_rust/flyteidl.core.rs index 0cfc6a19ae..2d9ee08fe1 100644 --- a/flyteidl/gen/pb_rust/flyteidl.core.rs +++ b/flyteidl/gen/pb_rust/flyteidl.core.rs @@ -2736,6 +2736,9 @@ pub struct TaskNodeOverrides { /// Override for the image used by task pods. #[prost(string, tag="3")] pub container_image: ::prost::alloc::string::String, + /// Override for the pod template used by task pods + #[prost(message, optional, tag="4")] + pub pod_template: ::core::option::Option, } /// A structure that uniquely identifies a launch plan in the system. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/flyteidl/protos/flyteidl/core/workflow.proto b/flyteidl/protos/flyteidl/core/workflow.proto index cf75850ece..6ba0c42c27 100644 --- a/flyteidl/protos/flyteidl/core/workflow.proto +++ b/flyteidl/protos/flyteidl/core/workflow.proto @@ -319,6 +319,10 @@ message TaskNodeOverrides { // Override for the image used by task pods. string container_image = 3; + + // Override for the pod template used by task pods + K8sPod pod_template = 4; + } // A structure that uniquely identifies a launch plan in the system. diff --git a/flyteplugins/go/tasks/pluginmachinery/core/exec_metadata.go b/flyteplugins/go/tasks/pluginmachinery/core/exec_metadata.go index 9a020bd188..f92a2a388c 100644 --- a/flyteplugins/go/tasks/pluginmachinery/core/exec_metadata.go +++ b/flyteplugins/go/tasks/pluginmachinery/core/exec_metadata.go @@ -13,6 +13,7 @@ type TaskOverrides interface { GetResources() *v1.ResourceRequirements GetExtendedResources() *core.ExtendedResources GetContainerImage() string + GetPodTemplate() *core.K8SPod GetConfig() *v1.ConfigMap } diff --git a/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_overrides.go b/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_overrides.go index ab42b1189f..af800591dd 100644 --- a/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_overrides.go +++ b/flyteplugins/go/tasks/pluginmachinery/core/mocks/task_overrides.go @@ -114,6 +114,40 @@ func (_m *TaskOverrides) GetExtendedResources() *flyteidlcore.ExtendedResources return r0 } +type TaskOverrides_GetPodTemplate struct { + *mock.Call +} + +func (_m TaskOverrides_GetPodTemplate) Return(_a0 *flyteidlcore.K8SPod) *TaskOverrides_GetPodTemplate { + return &TaskOverrides_GetPodTemplate{Call: _m.Call.Return(_a0)} +} + +func (_m *TaskOverrides) OnGetPodTemplate() *TaskOverrides_GetPodTemplate { + c_call := _m.On("GetPodTemplate") + return &TaskOverrides_GetPodTemplate{Call: c_call} +} + +func (_m *TaskOverrides) OnGetPodTemplateMatch(matchers ...interface{}) *TaskOverrides_GetPodTemplate { + c_call := _m.On("GetPodTemplate", matchers...) + return &TaskOverrides_GetPodTemplate{Call: c_call} +} + +// GetPodTemplate provides a mock function with given fields: +func (_m *TaskOverrides) GetPodTemplate() *flyteidlcore.K8SPod { + ret := _m.Called() + + var r0 *flyteidlcore.K8SPod + if rf, ok := ret.Get(0).(func() *flyteidlcore.K8SPod); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flyteidlcore.K8SPod) + } + } + + return r0 +} + type TaskOverrides_GetResources struct { *mock.Call } diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags.go index caa485ff39..7e94782749 100755 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags.go @@ -70,5 +70,6 @@ func (cfg K8sPluginConfig) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "update-base-backoff-duration"), defaultK8sConfig.UpdateBaseBackoffDuration, "Initial delay in exponential backoff when updating a resource in milliseconds.") cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "update-backoff-retries"), defaultK8sConfig.UpdateBackoffRetries, "Number of retries for exponential backoff when updating a resource.") cmdFlags.StringSlice(fmt.Sprintf("%v%v", prefix, "add-tolerations-for-extended-resources"), defaultK8sConfig.AddTolerationsForExtendedResources, "Name of the extended resources for which tolerations should be added.") + cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "enable-distributed-error-aggregation"), defaultK8sConfig.EnableDistributedErrorAggregation, "If true, will aggregate errors of different worker pods for distributed tasks.") return cmdFlags } diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags_test.go index cb50078620..3f68cfa1d7 100755 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags_test.go @@ -379,4 +379,18 @@ func TestK8sPluginConfig_SetFlags(t *testing.T) { } }) }) + t.Run("Test_enable-distributed-error-aggregation", func(t *testing.T) { + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("enable-distributed-error-aggregation", testValue) + if vBool, err := cmdFlags.GetBool("enable-distributed-error-aggregation"); err == nil { + testDecodeJson_K8sPluginConfig(t, fmt.Sprintf("%v", vBool), &actual.EnableDistributedErrorAggregation) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) } diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/plugin_exec_context.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/plugin_exec_context.go index ff9ae2ba9f..59bdd60550 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/plugin_exec_context.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/plugin_exec_context.go @@ -31,6 +31,10 @@ func (to *pluginTaskOverrides) GetContainerImage() string { return to.TaskOverrides.GetContainerImage() } +func (to *pluginTaskOverrides) GetPodTemplate() *core.K8SPod { + return to.TaskOverrides.GetPodTemplate() +} + type pluginTaskExecutionMetadata struct { pluginsCore.TaskExecutionMetadata interruptible *bool diff --git a/flyteplugins/tests/end_to_end.go b/flyteplugins/tests/end_to_end.go index b045deae13..cef7a2932f 100644 --- a/flyteplugins/tests/end_to_end.go +++ b/flyteplugins/tests/end_to_end.go @@ -148,6 +148,7 @@ func RunPluginEndToEndTest(t *testing.T, executor pluginCore.Plugin, template *i }) overrides.OnGetExtendedResources().Return(&idlCore.ExtendedResources{}) overrides.OnGetContainerImage().Return("") + overrides.OnGetPodTemplate().Return(&idlCore.K8SPod{}) tMeta := &coreMocks.TaskExecutionMetadata{} tMeta.OnGetTaskExecutionID().Return(tID) diff --git a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/iface.go b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/iface.go index c2022dea25..669f6ed10e 100644 --- a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/iface.go +++ b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/iface.go @@ -443,6 +443,7 @@ type ExecutableNode interface { IsInterruptible() *bool GetName() string GetContainerImage() string + GetPodTemplate() *core.K8SPod } // ExecutableWorkflowStatus is an interface for the Workflow p. This is the mutable portion for a Workflow diff --git a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/nodes.go b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/nodes.go index 743327e373..4bbbd34552 100644 --- a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/nodes.go +++ b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/nodes.go @@ -159,6 +159,8 @@ type NodeSpec struct { Interruptible *bool `json:"interruptible,omitempty"` ContainerImage string `json:"containerImage,omitempty"` + + PodTemplate *core.K8SPod `json:"podTemplate,omitempty" protobuf:"bytes,23,opt,name=podTemplate"` } func (in *NodeSpec) GetName() string { @@ -268,3 +270,7 @@ func (in *NodeSpec) GetInputBindings() []*Binding { func (in *NodeSpec) GetContainerImage() string { return in.ContainerImage } + +func (in *NodeSpec) GetPodTemplate() *core.K8SPod { + return in.PodTemplate +} diff --git a/flytepropeller/pkg/compiler/transformers/k8s/node.go b/flytepropeller/pkg/compiler/transformers/k8s/node.go index 18ec1ba02f..2c13c86c5a 100644 --- a/flytepropeller/pkg/compiler/transformers/k8s/node.go +++ b/flytepropeller/pkg/compiler/transformers/k8s/node.go @@ -31,6 +31,7 @@ func buildNodeSpec(n *core.Node, tasks []*core.CompiledTask, errs errors.Compile var resources *core.Resources var extendedResources *v1alpha1.ExtendedResources var containerImage string + var podtemplate *core.K8SPod if n.GetTaskNode() != nil { taskID := n.GetTaskNode().GetReferenceId().String() // TODO: Use task index for quick lookup @@ -60,6 +61,10 @@ func buildNodeSpec(n *core.Node, tasks []*core.CompiledTask, errs errors.Compile if len(overrides.GetContainerImage()) > 0 { containerImage = overrides.GetContainerImage() } + + if overrides.GetPodTemplate() != nil { + podtemplate = overrides.GetPodTemplate() + } } } @@ -102,6 +107,7 @@ func buildNodeSpec(n *core.Node, tasks []*core.CompiledTask, errs errors.Compile ActiveDeadline: activeDeadline, Interruptible: interruptible, ContainerImage: containerImage, + PodTemplate: podtemplate, } switch v := n.GetTarget().(type) { From f4d7278f322a9e598cd777bd7bc6e1d33cd1cd6d Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 14 Dec 2024 00:40:23 +0800 Subject: [PATCH 02/13] test Signed-off-by: Nelson Chen --- flyteidl/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flyteidl/setup.py b/flyteidl/setup.py index 1030e43304..6a93e4e830 100644 --- a/flyteidl/setup.py +++ b/flyteidl/setup.py @@ -1,3 +1,3 @@ from setuptools import setup, find_packages -setup() +setup(1.13.9) From c4dad5811e494e7b68cf668ed111c7152cfe74fe Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 14 Dec 2024 00:41:25 +0800 Subject: [PATCH 03/13] test Signed-off-by: Nelson Chen --- flyteidl/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flyteidl/setup.py b/flyteidl/setup.py index 6a93e4e830..e633f16321 100644 --- a/flyteidl/setup.py +++ b/flyteidl/setup.py @@ -1,3 +1,3 @@ from setuptools import setup, find_packages -setup(1.13.9) +setup(version='1.13.9') \ No newline at end of file From 092ac390b5d53879ff2557d4b42db4ee9f3864a2 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Fri, 20 Dec 2024 04:58:46 +0800 Subject: [PATCH 04/13] podtemplate override Signed-off-by: Nelson Chen --- flyteidl/clients/go/assets/admin.swagger.json | 3 + flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts | 6 ++ flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go | 96 +++++++++++-------- .../flyteidl/service/admin.swagger.json | 3 + .../flyteidl/service/agent.swagger.json | 3 + .../external_plugin_service.swagger.json | 3 + flyteidl/gen/pb-js/flyteidl.d.ts | 6 ++ flyteidl/gen/pb-js/flyteidl.js | 17 ++++ .../gen/pb_python/flyteidl/core/tasks_pb2.py | 24 ++--- .../gen/pb_python/flyteidl/core/tasks_pb2.pyi | 6 +- flyteidl/gen/pb_rust/flyteidl.core.rs | 2 + flyteidl/protos/flyteidl/core/tasks.proto | 2 + flyteidl/setup.py | 2 +- .../pluginmachinery/flytek8s/pod_helper.go | 33 ++++++- 14 files changed, 148 insertions(+), 58 deletions(-) diff --git a/flyteidl/clients/go/assets/admin.swagger.json b/flyteidl/clients/go/assets/admin.swagger.json index 8c450b3ac4..f320a989c2 100644 --- a/flyteidl/clients/go/assets/admin.swagger.json +++ b/flyteidl/clients/go/assets/admin.swagger.json @@ -7377,6 +7377,9 @@ "data_config": { "$ref": "#/definitions/coreDataLoadingConfig", "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" + }, + "primarycontainername": { + "type": "string" } }, "description": "Defines a pod spec and additional pod metadata that is created when a task is executed." diff --git a/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts b/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts index eb2156cce7..6dbdac1521 100644 --- a/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts +++ b/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts @@ -1101,6 +1101,11 @@ export class K8sPod extends Message { */ dataConfig?: DataLoadingConfig; + /** + * @generated from field: string primarycontainername = 4; + */ + primarycontainername = ""; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -1112,6 +1117,7 @@ export class K8sPod extends Message { { no: 1, name: "metadata", kind: "message", T: K8sObjectMetadata }, { no: 2, name: "pod_spec", kind: "message", T: Struct }, { no: 3, name: "data_config", kind: "message", T: DataLoadingConfig }, + { no: 4, name: "primarycontainername", kind: "scalar", T: 9 /* ScalarType.STRING */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): K8sPod { diff --git a/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go b/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go index ceb9c71f1f..cfea868473 100644 --- a/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go @@ -1402,7 +1402,8 @@ type K8SPod struct { // are not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation // to understand the default paths. // Only K8s - DataConfig *DataLoadingConfig `protobuf:"bytes,3,opt,name=data_config,json=dataConfig,proto3" json:"data_config,omitempty"` + DataConfig *DataLoadingConfig `protobuf:"bytes,3,opt,name=data_config,json=dataConfig,proto3" json:"data_config,omitempty"` + Primarycontainername string `protobuf:"bytes,4,opt,name=primarycontainername,proto3" json:"primarycontainername,omitempty"` } func (x *K8SPod) Reset() { @@ -1458,6 +1459,13 @@ func (x *K8SPod) GetDataConfig() *DataLoadingConfig { return nil } +func (x *K8SPod) GetPrimarycontainername() string { + if x != nil { + return x.Primarycontainername + } + return "" +} + // Metadata for building a kubernetes object when a task is executed. type K8SObjectMetadata struct { state protoimpl.MessageState @@ -1866,7 +1874,7 @@ var file_flyteidl_core_tasks_proto_rawDesc = []byte{ 0x31, 0x0a, 0x10, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x61, 0x70, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x59, 0x41, 0x4d, 0x4c, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x52, 0x4f, 0x54, 0x4f, - 0x10, 0x02, 0x22, 0xbd, 0x01, 0x0a, 0x06, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x12, 0x3c, 0x0a, + 0x10, 0x02, 0x22, 0xf1, 0x01, 0x0a, 0x06, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x12, 0x3c, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4b, 0x38, 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, @@ -1878,46 +1886,50 @@ var file_flyteidl_core_tasks_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x4c, 0x6f, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x22, 0xa9, 0x02, 0x0a, 0x11, 0x4b, 0x38, 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x44, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, - 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, - 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4b, 0x38, 0x73, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x53, - 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x4b, 0x38, 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3e, - 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x92, - 0x01, 0x0a, 0x03, 0x53, 0x71, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x64, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x71, 0x6c, 0x2e, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x63, - 0x74, 0x52, 0x07, 0x64, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x22, 0x37, 0x0a, 0x07, 0x44, 0x69, - 0x61, 0x6c, 0x65, 0x63, 0x74, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4e, 0x53, 0x49, 0x10, 0x01, 0x12, 0x08, - 0x0a, 0x04, 0x48, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x4f, 0x54, 0x48, 0x45, - 0x52, 0x10, 0x03, 0x42, 0xb0, 0x01, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x2e, 0x66, 0x6c, 0x79, 0x74, - 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0a, 0x54, 0x61, 0x73, 0x6b, 0x73, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x6f, 0x72, 0x67, 0x2f, 0x66, 0x6c, 0x79, - 0x74, 0x65, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, - 0x70, 0x62, 0x2d, 0x67, 0x6f, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x63, - 0x6f, 0x72, 0x65, 0xa2, 0x02, 0x03, 0x46, 0x43, 0x58, 0xaa, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, - 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0xca, 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, - 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0xe2, 0x02, 0x19, 0x46, 0x6c, 0x79, 0x74, - 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0e, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, - 0x3a, 0x3a, 0x43, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x69, 0x67, 0x12, 0x32, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, + 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xa9, 0x02, 0x0a, 0x11, 0x4b, 0x38, 0x73, 0x4f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x44, 0x0a, 0x06, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x66, + 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4b, 0x38, 0x73, + 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, + 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x12, 0x53, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, + 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4b, 0x38, 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, + 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x92, 0x01, 0x0a, 0x03, 0x53, 0x71, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x74, + 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x64, 0x69, 0x61, 0x6c, + 0x65, 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, + 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x71, 0x6c, 0x2e, 0x44, 0x69, + 0x61, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x07, 0x64, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x22, 0x37, + 0x0a, 0x07, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x44, + 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4e, 0x53, 0x49, + 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, + 0x4f, 0x54, 0x48, 0x45, 0x52, 0x10, 0x03, 0x42, 0xb0, 0x01, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x2e, + 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0a, 0x54, + 0x61, 0x73, 0x6b, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3a, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x6f, 0x72, 0x67, + 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, + 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x62, 0x2d, 0x67, 0x6f, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, + 0x64, 0x6c, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0xa2, 0x02, 0x03, 0x46, 0x43, 0x58, 0xaa, 0x02, 0x0d, + 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0xca, 0x02, 0x0d, + 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0xe2, 0x02, 0x19, + 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0x5c, 0x47, 0x50, + 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0e, 0x46, 0x6c, 0x79, 0x74, + 0x65, 0x69, 0x64, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json index 8c450b3ac4..f320a989c2 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json @@ -7377,6 +7377,9 @@ "data_config": { "$ref": "#/definitions/coreDataLoadingConfig", "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" + }, + "primarycontainername": { + "type": "string" } }, "description": "Defines a pod spec and additional pod metadata that is created when a task is executed." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json index eae5bd2df7..e725acfc45 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json @@ -1256,6 +1256,9 @@ "data_config": { "$ref": "#/definitions/coreDataLoadingConfig", "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" + }, + "primarycontainername": { + "type": "string" } }, "description": "Defines a pod spec and additional pod metadata that is created when a task is executed." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json index 1880700dc2..65e2809e95 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json @@ -531,6 +531,9 @@ "data_config": { "$ref": "#/definitions/coreDataLoadingConfig", "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" + }, + "primarycontainername": { + "type": "string" } }, "description": "Defines a pod spec and additional pod metadata that is created when a task is executed." diff --git a/flyteidl/gen/pb-js/flyteidl.d.ts b/flyteidl/gen/pb-js/flyteidl.d.ts index d80e01ddfe..9e73fb7a2d 100644 --- a/flyteidl/gen/pb-js/flyteidl.d.ts +++ b/flyteidl/gen/pb-js/flyteidl.d.ts @@ -6959,6 +6959,9 @@ export namespace flyteidl { /** K8sPod dataConfig */ dataConfig?: (flyteidl.core.IDataLoadingConfig|null); + + /** K8sPod primarycontainername */ + primarycontainername?: (string|null); } /** Represents a K8sPod. */ @@ -6979,6 +6982,9 @@ export namespace flyteidl { /** K8sPod dataConfig. */ public dataConfig?: (flyteidl.core.IDataLoadingConfig|null); + /** K8sPod primarycontainername. */ + public primarycontainername: string; + /** * Creates a new K8sPod instance using the specified properties. * @param [properties] Properties to set diff --git a/flyteidl/gen/pb-js/flyteidl.js b/flyteidl/gen/pb-js/flyteidl.js index a8722e400c..b931baea2b 100644 --- a/flyteidl/gen/pb-js/flyteidl.js +++ b/flyteidl/gen/pb-js/flyteidl.js @@ -16850,6 +16850,7 @@ * @property {flyteidl.core.IK8sObjectMetadata|null} [metadata] K8sPod metadata * @property {google.protobuf.IStruct|null} [podSpec] K8sPod podSpec * @property {flyteidl.core.IDataLoadingConfig|null} [dataConfig] K8sPod dataConfig + * @property {string|null} [primarycontainername] K8sPod primarycontainername */ /** @@ -16891,6 +16892,14 @@ */ K8sPod.prototype.dataConfig = null; + /** + * K8sPod primarycontainername. + * @member {string} primarycontainername + * @memberof flyteidl.core.K8sPod + * @instance + */ + K8sPod.prototype.primarycontainername = ""; + /** * Creates a new K8sPod instance using the specified properties. * @function create @@ -16921,6 +16930,8 @@ $root.google.protobuf.Struct.encode(message.podSpec, writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim(); if (message.dataConfig != null && message.hasOwnProperty("dataConfig")) $root.flyteidl.core.DataLoadingConfig.encode(message.dataConfig, writer.uint32(/* id 3, wireType 2 =*/26).fork()).ldelim(); + if (message.primarycontainername != null && message.hasOwnProperty("primarycontainername")) + writer.uint32(/* id 4, wireType 2 =*/34).string(message.primarycontainername); return writer; }; @@ -16951,6 +16962,9 @@ case 3: message.dataConfig = $root.flyteidl.core.DataLoadingConfig.decode(reader, reader.uint32()); break; + case 4: + message.primarycontainername = reader.string(); + break; default: reader.skipType(tag & 7); break; @@ -16985,6 +16999,9 @@ if (error) return "dataConfig." + error; } + if (message.primarycontainername != null && message.hasOwnProperty("primarycontainername")) + if (!$util.isString(message.primarycontainername)) + return "primarycontainername: string expected"; return null; }; diff --git a/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.py b/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.py index 43beeeeca2..762128263b 100644 --- a/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.py +++ b/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.py @@ -19,7 +19,7 @@ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19\x66lyteidl/core/tasks.proto\x12\rflyteidl.core\x1a\x1e\x66lyteidl/core/identifier.proto\x1a\x1d\x66lyteidl/core/interface.proto\x1a\x1c\x66lyteidl/core/literals.proto\x1a\x1c\x66lyteidl/core/security.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xd0\x02\n\tResources\x12\x42\n\x08requests\x18\x01 \x03(\x0b\x32&.flyteidl.core.Resources.ResourceEntryR\x08requests\x12>\n\x06limits\x18\x02 \x03(\x0b\x32&.flyteidl.core.Resources.ResourceEntryR\x06limits\x1a`\n\rResourceEntry\x12\x39\n\x04name\x18\x01 \x01(\x0e\x32%.flyteidl.core.Resources.ResourceNameR\x04name\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\"]\n\x0cResourceName\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x07\n\x03\x43PU\x10\x01\x12\x07\n\x03GPU\x10\x02\x12\n\n\x06MEMORY\x10\x03\x12\x0b\n\x07STORAGE\x10\x04\x12\x15\n\x11\x45PHEMERAL_STORAGE\x10\x05\"\x91\x01\n\x0eGPUAccelerator\x12\x16\n\x06\x64\x65vice\x18\x01 \x01(\tR\x06\x64\x65vice\x12&\n\runpartitioned\x18\x02 \x01(\x08H\x00R\runpartitioned\x12\'\n\x0epartition_size\x18\x03 \x01(\tH\x00R\rpartitionSizeB\x16\n\x14partition_size_value\"[\n\x11\x45xtendedResources\x12\x46\n\x0fgpu_accelerator\x18\x01 \x01(\x0b\x32\x1d.flyteidl.core.GPUAcceleratorR\x0egpuAccelerator\"\xac\x01\n\x0fRuntimeMetadata\x12>\n\x04type\x18\x01 \x01(\x0e\x32*.flyteidl.core.RuntimeMetadata.RuntimeTypeR\x04type\x12\x18\n\x07version\x18\x02 \x01(\tR\x07version\x12\x16\n\x06\x66lavor\x18\x03 \x01(\tR\x06\x66lavor\"\'\n\x0bRuntimeType\x12\t\n\x05OTHER\x10\x00\x12\r\n\tFLYTE_SDK\x10\x01\"\xc7\x05\n\x0cTaskMetadata\x12\"\n\x0c\x64iscoverable\x18\x01 \x01(\x08R\x0c\x64iscoverable\x12\x38\n\x07runtime\x18\x02 \x01(\x0b\x32\x1e.flyteidl.core.RuntimeMetadataR\x07runtime\x12\x33\n\x07timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\x07timeout\x12\x36\n\x07retries\x18\x05 \x01(\x0b\x32\x1c.flyteidl.core.RetryStrategyR\x07retries\x12+\n\x11\x64iscovery_version\x18\x06 \x01(\tR\x10\x64iscoveryVersion\x12\x38\n\x18\x64\x65precated_error_message\x18\x07 \x01(\tR\x16\x64\x65precatedErrorMessage\x12&\n\rinterruptible\x18\x08 \x01(\x08H\x00R\rinterruptible\x12-\n\x12\x63\x61\x63he_serializable\x18\t \x01(\x08R\x11\x63\x61\x63heSerializable\x12%\n\x0egenerates_deck\x18\n \x01(\x08R\rgeneratesDeck\x12\x39\n\x04tags\x18\x0b \x03(\x0b\x32%.flyteidl.core.TaskMetadata.TagsEntryR\x04tags\x12*\n\x11pod_template_name\x18\x0c \x01(\tR\x0fpodTemplateName\x12\x35\n\x17\x63\x61\x63he_ignore_input_vars\x18\r \x03(\tR\x14\x63\x61\x63heIgnoreInputVars\x12\x19\n\x08is_eager\x18\x0e \x01(\x08R\x07isEager\x1a\x37\n\tTagsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x15\n\x13interruptible_value\"\xd6\x05\n\x0cTaskTemplate\x12)\n\x02id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierR\x02id\x12\x12\n\x04type\x18\x02 \x01(\tR\x04type\x12\x37\n\x08metadata\x18\x03 \x01(\x0b\x32\x1b.flyteidl.core.TaskMetadataR\x08metadata\x12;\n\tinterface\x18\x04 \x01(\x0b\x32\x1d.flyteidl.core.TypedInterfaceR\tinterface\x12/\n\x06\x63ustom\x18\x05 \x01(\x0b\x32\x17.google.protobuf.StructR\x06\x63ustom\x12\x38\n\tcontainer\x18\x06 \x01(\x0b\x32\x18.flyteidl.core.ContainerH\x00R\tcontainer\x12\x30\n\x07k8s_pod\x18\x11 \x01(\x0b\x32\x15.flyteidl.core.K8sPodH\x00R\x06k8sPod\x12&\n\x03sql\x18\x12 \x01(\x0b\x32\x12.flyteidl.core.SqlH\x00R\x03sql\x12*\n\x11task_type_version\x18\x07 \x01(\x05R\x0ftaskTypeVersion\x12I\n\x10security_context\x18\x08 \x01(\x0b\x32\x1e.flyteidl.core.SecurityContextR\x0fsecurityContext\x12O\n\x12\x65xtended_resources\x18\t \x01(\x0b\x32 .flyteidl.core.ExtendedResourcesR\x11\x65xtendedResources\x12?\n\x06\x63onfig\x18\x10 \x03(\x0b\x32\'.flyteidl.core.TaskTemplate.ConfigEntryR\x06\x63onfig\x1a\x39\n\x0b\x43onfigEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x08\n\x06target\"6\n\rContainerPort\x12%\n\x0e\x63ontainer_port\x18\x01 \x01(\rR\rcontainerPort\"\xfc\x03\n\tContainer\x12\x14\n\x05image\x18\x01 \x01(\tR\x05image\x12\x18\n\x07\x63ommand\x18\x02 \x03(\tR\x07\x63ommand\x12\x12\n\x04\x61rgs\x18\x03 \x03(\tR\x04\x61rgs\x12\x36\n\tresources\x18\x04 \x01(\x0b\x32\x18.flyteidl.core.ResourcesR\tresources\x12-\n\x03\x65nv\x18\x05 \x03(\x0b\x32\x1b.flyteidl.core.KeyValuePairR\x03\x65nv\x12\x37\n\x06\x63onfig\x18\x06 \x03(\x0b\x32\x1b.flyteidl.core.KeyValuePairB\x02\x18\x01R\x06\x63onfig\x12\x32\n\x05ports\x18\x07 \x03(\x0b\x32\x1c.flyteidl.core.ContainerPortR\x05ports\x12\x41\n\x0b\x64\x61ta_config\x18\t \x01(\x0b\x32 .flyteidl.core.DataLoadingConfigR\ndataConfig\x12I\n\x0c\x61rchitecture\x18\n \x01(\x0e\x32%.flyteidl.core.Container.ArchitectureR\x0c\x61rchitecture\"I\n\x0c\x41rchitecture\x12\x0b\n\x07UNKNOWN\x10\x00\x12\t\n\x05\x41MD64\x10\x01\x12\t\n\x05\x41RM64\x10\x02\x12\n\n\x06\x41RM_V6\x10\x03\x12\n\n\x06\x41RM_V7\x10\x04\"\xb5\x02\n\nIOStrategy\x12K\n\rdownload_mode\x18\x01 \x01(\x0e\x32&.flyteidl.core.IOStrategy.DownloadModeR\x0c\x64ownloadMode\x12\x45\n\x0bupload_mode\x18\x02 \x01(\x0e\x32$.flyteidl.core.IOStrategy.UploadModeR\nuploadMode\"L\n\x0c\x44ownloadMode\x12\x12\n\x0e\x44OWNLOAD_EAGER\x10\x00\x12\x13\n\x0f\x44OWNLOAD_STREAM\x10\x01\x12\x13\n\x0f\x44O_NOT_DOWNLOAD\x10\x02\"E\n\nUploadMode\x12\x12\n\x0eUPLOAD_ON_EXIT\x10\x00\x12\x10\n\x0cUPLOAD_EAGER\x10\x01\x12\x11\n\rDO_NOT_UPLOAD\x10\x02\"\xa7\x02\n\x11\x44\x61taLoadingConfig\x12\x18\n\x07\x65nabled\x18\x01 \x01(\x08R\x07\x65nabled\x12\x1d\n\ninput_path\x18\x02 \x01(\tR\tinputPath\x12\x1f\n\x0boutput_path\x18\x03 \x01(\tR\noutputPath\x12I\n\x06\x66ormat\x18\x04 \x01(\x0e\x32\x31.flyteidl.core.DataLoadingConfig.LiteralMapFormatR\x06\x66ormat\x12:\n\x0bio_strategy\x18\x05 \x01(\x0b\x32\x19.flyteidl.core.IOStrategyR\nioStrategy\"1\n\x10LiteralMapFormat\x12\x08\n\x04JSON\x10\x00\x12\x08\n\x04YAML\x10\x01\x12\t\n\x05PROTO\x10\x02\"\xbd\x01\n\x06K8sPod\x12<\n\x08metadata\x18\x01 \x01(\x0b\x32 .flyteidl.core.K8sObjectMetadataR\x08metadata\x12\x32\n\x08pod_spec\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x07podSpec\x12\x41\n\x0b\x64\x61ta_config\x18\x03 \x01(\x0b\x32 .flyteidl.core.DataLoadingConfigR\ndataConfig\"\xa9\x02\n\x11K8sObjectMetadata\x12\x44\n\x06labels\x18\x01 \x03(\x0b\x32,.flyteidl.core.K8sObjectMetadata.LabelsEntryR\x06labels\x12S\n\x0b\x61nnotations\x18\x02 \x03(\x0b\x32\x31.flyteidl.core.K8sObjectMetadata.AnnotationsEntryR\x0b\x61nnotations\x1a\x39\n\x0bLabelsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a>\n\x10\x41nnotationsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"\x92\x01\n\x03Sql\x12\x1c\n\tstatement\x18\x01 \x01(\tR\tstatement\x12\x34\n\x07\x64ialect\x18\x02 \x01(\x0e\x32\x1a.flyteidl.core.Sql.DialectR\x07\x64ialect\"7\n\x07\x44ialect\x12\r\n\tUNDEFINED\x10\x00\x12\x08\n\x04\x41NSI\x10\x01\x12\x08\n\x04HIVE\x10\x02\x12\t\n\x05OTHER\x10\x03\x42\xb0\x01\n\x11\x63om.flyteidl.coreB\nTasksProtoP\x01Z:github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core\xa2\x02\x03\x46\x43X\xaa\x02\rFlyteidl.Core\xca\x02\rFlyteidl\\Core\xe2\x02\x19\x46lyteidl\\Core\\GPBMetadata\xea\x02\x0e\x46lyteidl::Coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19\x66lyteidl/core/tasks.proto\x12\rflyteidl.core\x1a\x1e\x66lyteidl/core/identifier.proto\x1a\x1d\x66lyteidl/core/interface.proto\x1a\x1c\x66lyteidl/core/literals.proto\x1a\x1c\x66lyteidl/core/security.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xd0\x02\n\tResources\x12\x42\n\x08requests\x18\x01 \x03(\x0b\x32&.flyteidl.core.Resources.ResourceEntryR\x08requests\x12>\n\x06limits\x18\x02 \x03(\x0b\x32&.flyteidl.core.Resources.ResourceEntryR\x06limits\x1a`\n\rResourceEntry\x12\x39\n\x04name\x18\x01 \x01(\x0e\x32%.flyteidl.core.Resources.ResourceNameR\x04name\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\"]\n\x0cResourceName\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x07\n\x03\x43PU\x10\x01\x12\x07\n\x03GPU\x10\x02\x12\n\n\x06MEMORY\x10\x03\x12\x0b\n\x07STORAGE\x10\x04\x12\x15\n\x11\x45PHEMERAL_STORAGE\x10\x05\"\x91\x01\n\x0eGPUAccelerator\x12\x16\n\x06\x64\x65vice\x18\x01 \x01(\tR\x06\x64\x65vice\x12&\n\runpartitioned\x18\x02 \x01(\x08H\x00R\runpartitioned\x12\'\n\x0epartition_size\x18\x03 \x01(\tH\x00R\rpartitionSizeB\x16\n\x14partition_size_value\"[\n\x11\x45xtendedResources\x12\x46\n\x0fgpu_accelerator\x18\x01 \x01(\x0b\x32\x1d.flyteidl.core.GPUAcceleratorR\x0egpuAccelerator\"\xac\x01\n\x0fRuntimeMetadata\x12>\n\x04type\x18\x01 \x01(\x0e\x32*.flyteidl.core.RuntimeMetadata.RuntimeTypeR\x04type\x12\x18\n\x07version\x18\x02 \x01(\tR\x07version\x12\x16\n\x06\x66lavor\x18\x03 \x01(\tR\x06\x66lavor\"\'\n\x0bRuntimeType\x12\t\n\x05OTHER\x10\x00\x12\r\n\tFLYTE_SDK\x10\x01\"\xc7\x05\n\x0cTaskMetadata\x12\"\n\x0c\x64iscoverable\x18\x01 \x01(\x08R\x0c\x64iscoverable\x12\x38\n\x07runtime\x18\x02 \x01(\x0b\x32\x1e.flyteidl.core.RuntimeMetadataR\x07runtime\x12\x33\n\x07timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\x07timeout\x12\x36\n\x07retries\x18\x05 \x01(\x0b\x32\x1c.flyteidl.core.RetryStrategyR\x07retries\x12+\n\x11\x64iscovery_version\x18\x06 \x01(\tR\x10\x64iscoveryVersion\x12\x38\n\x18\x64\x65precated_error_message\x18\x07 \x01(\tR\x16\x64\x65precatedErrorMessage\x12&\n\rinterruptible\x18\x08 \x01(\x08H\x00R\rinterruptible\x12-\n\x12\x63\x61\x63he_serializable\x18\t \x01(\x08R\x11\x63\x61\x63heSerializable\x12%\n\x0egenerates_deck\x18\n \x01(\x08R\rgeneratesDeck\x12\x39\n\x04tags\x18\x0b \x03(\x0b\x32%.flyteidl.core.TaskMetadata.TagsEntryR\x04tags\x12*\n\x11pod_template_name\x18\x0c \x01(\tR\x0fpodTemplateName\x12\x35\n\x17\x63\x61\x63he_ignore_input_vars\x18\r \x03(\tR\x14\x63\x61\x63heIgnoreInputVars\x12\x19\n\x08is_eager\x18\x0e \x01(\x08R\x07isEager\x1a\x37\n\tTagsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x15\n\x13interruptible_value\"\xd6\x05\n\x0cTaskTemplate\x12)\n\x02id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierR\x02id\x12\x12\n\x04type\x18\x02 \x01(\tR\x04type\x12\x37\n\x08metadata\x18\x03 \x01(\x0b\x32\x1b.flyteidl.core.TaskMetadataR\x08metadata\x12;\n\tinterface\x18\x04 \x01(\x0b\x32\x1d.flyteidl.core.TypedInterfaceR\tinterface\x12/\n\x06\x63ustom\x18\x05 \x01(\x0b\x32\x17.google.protobuf.StructR\x06\x63ustom\x12\x38\n\tcontainer\x18\x06 \x01(\x0b\x32\x18.flyteidl.core.ContainerH\x00R\tcontainer\x12\x30\n\x07k8s_pod\x18\x11 \x01(\x0b\x32\x15.flyteidl.core.K8sPodH\x00R\x06k8sPod\x12&\n\x03sql\x18\x12 \x01(\x0b\x32\x12.flyteidl.core.SqlH\x00R\x03sql\x12*\n\x11task_type_version\x18\x07 \x01(\x05R\x0ftaskTypeVersion\x12I\n\x10security_context\x18\x08 \x01(\x0b\x32\x1e.flyteidl.core.SecurityContextR\x0fsecurityContext\x12O\n\x12\x65xtended_resources\x18\t \x01(\x0b\x32 .flyteidl.core.ExtendedResourcesR\x11\x65xtendedResources\x12?\n\x06\x63onfig\x18\x10 \x03(\x0b\x32\'.flyteidl.core.TaskTemplate.ConfigEntryR\x06\x63onfig\x1a\x39\n\x0b\x43onfigEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x08\n\x06target\"6\n\rContainerPort\x12%\n\x0e\x63ontainer_port\x18\x01 \x01(\rR\rcontainerPort\"\xfc\x03\n\tContainer\x12\x14\n\x05image\x18\x01 \x01(\tR\x05image\x12\x18\n\x07\x63ommand\x18\x02 \x03(\tR\x07\x63ommand\x12\x12\n\x04\x61rgs\x18\x03 \x03(\tR\x04\x61rgs\x12\x36\n\tresources\x18\x04 \x01(\x0b\x32\x18.flyteidl.core.ResourcesR\tresources\x12-\n\x03\x65nv\x18\x05 \x03(\x0b\x32\x1b.flyteidl.core.KeyValuePairR\x03\x65nv\x12\x37\n\x06\x63onfig\x18\x06 \x03(\x0b\x32\x1b.flyteidl.core.KeyValuePairB\x02\x18\x01R\x06\x63onfig\x12\x32\n\x05ports\x18\x07 \x03(\x0b\x32\x1c.flyteidl.core.ContainerPortR\x05ports\x12\x41\n\x0b\x64\x61ta_config\x18\t \x01(\x0b\x32 .flyteidl.core.DataLoadingConfigR\ndataConfig\x12I\n\x0c\x61rchitecture\x18\n \x01(\x0e\x32%.flyteidl.core.Container.ArchitectureR\x0c\x61rchitecture\"I\n\x0c\x41rchitecture\x12\x0b\n\x07UNKNOWN\x10\x00\x12\t\n\x05\x41MD64\x10\x01\x12\t\n\x05\x41RM64\x10\x02\x12\n\n\x06\x41RM_V6\x10\x03\x12\n\n\x06\x41RM_V7\x10\x04\"\xb5\x02\n\nIOStrategy\x12K\n\rdownload_mode\x18\x01 \x01(\x0e\x32&.flyteidl.core.IOStrategy.DownloadModeR\x0c\x64ownloadMode\x12\x45\n\x0bupload_mode\x18\x02 \x01(\x0e\x32$.flyteidl.core.IOStrategy.UploadModeR\nuploadMode\"L\n\x0c\x44ownloadMode\x12\x12\n\x0e\x44OWNLOAD_EAGER\x10\x00\x12\x13\n\x0f\x44OWNLOAD_STREAM\x10\x01\x12\x13\n\x0f\x44O_NOT_DOWNLOAD\x10\x02\"E\n\nUploadMode\x12\x12\n\x0eUPLOAD_ON_EXIT\x10\x00\x12\x10\n\x0cUPLOAD_EAGER\x10\x01\x12\x11\n\rDO_NOT_UPLOAD\x10\x02\"\xa7\x02\n\x11\x44\x61taLoadingConfig\x12\x18\n\x07\x65nabled\x18\x01 \x01(\x08R\x07\x65nabled\x12\x1d\n\ninput_path\x18\x02 \x01(\tR\tinputPath\x12\x1f\n\x0boutput_path\x18\x03 \x01(\tR\noutputPath\x12I\n\x06\x66ormat\x18\x04 \x01(\x0e\x32\x31.flyteidl.core.DataLoadingConfig.LiteralMapFormatR\x06\x66ormat\x12:\n\x0bio_strategy\x18\x05 \x01(\x0b\x32\x19.flyteidl.core.IOStrategyR\nioStrategy\"1\n\x10LiteralMapFormat\x12\x08\n\x04JSON\x10\x00\x12\x08\n\x04YAML\x10\x01\x12\t\n\x05PROTO\x10\x02\"\xf1\x01\n\x06K8sPod\x12<\n\x08metadata\x18\x01 \x01(\x0b\x32 .flyteidl.core.K8sObjectMetadataR\x08metadata\x12\x32\n\x08pod_spec\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x07podSpec\x12\x41\n\x0b\x64\x61ta_config\x18\x03 \x01(\x0b\x32 .flyteidl.core.DataLoadingConfigR\ndataConfig\x12\x32\n\x14primarycontainername\x18\x04 \x01(\tR\x14primarycontainername\"\xa9\x02\n\x11K8sObjectMetadata\x12\x44\n\x06labels\x18\x01 \x03(\x0b\x32,.flyteidl.core.K8sObjectMetadata.LabelsEntryR\x06labels\x12S\n\x0b\x61nnotations\x18\x02 \x03(\x0b\x32\x31.flyteidl.core.K8sObjectMetadata.AnnotationsEntryR\x0b\x61nnotations\x1a\x39\n\x0bLabelsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a>\n\x10\x41nnotationsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"\x92\x01\n\x03Sql\x12\x1c\n\tstatement\x18\x01 \x01(\tR\tstatement\x12\x34\n\x07\x64ialect\x18\x02 \x01(\x0e\x32\x1a.flyteidl.core.Sql.DialectR\x07\x64ialect\"7\n\x07\x44ialect\x12\r\n\tUNDEFINED\x10\x00\x12\x08\n\x04\x41NSI\x10\x01\x12\x08\n\x04HIVE\x10\x02\x12\t\n\x05OTHER\x10\x03\x42\xb0\x01\n\x11\x63om.flyteidl.coreB\nTasksProtoP\x01Z:github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core\xa2\x02\x03\x46\x43X\xaa\x02\rFlyteidl.Core\xca\x02\rFlyteidl\\Core\xe2\x02\x19\x46lyteidl\\Core\\GPBMetadata\xea\x02\x0e\x46lyteidl::Coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -77,15 +77,15 @@ _globals['_DATALOADINGCONFIG_LITERALMAPFORMAT']._serialized_start=3553 _globals['_DATALOADINGCONFIG_LITERALMAPFORMAT']._serialized_end=3602 _globals['_K8SPOD']._serialized_start=3605 - _globals['_K8SPOD']._serialized_end=3794 - _globals['_K8SOBJECTMETADATA']._serialized_start=3797 - _globals['_K8SOBJECTMETADATA']._serialized_end=4094 - _globals['_K8SOBJECTMETADATA_LABELSENTRY']._serialized_start=3973 - _globals['_K8SOBJECTMETADATA_LABELSENTRY']._serialized_end=4030 - _globals['_K8SOBJECTMETADATA_ANNOTATIONSENTRY']._serialized_start=4032 - _globals['_K8SOBJECTMETADATA_ANNOTATIONSENTRY']._serialized_end=4094 - _globals['_SQL']._serialized_start=4097 - _globals['_SQL']._serialized_end=4243 - _globals['_SQL_DIALECT']._serialized_start=4188 - _globals['_SQL_DIALECT']._serialized_end=4243 + _globals['_K8SPOD']._serialized_end=3846 + _globals['_K8SOBJECTMETADATA']._serialized_start=3849 + _globals['_K8SOBJECTMETADATA']._serialized_end=4146 + _globals['_K8SOBJECTMETADATA_LABELSENTRY']._serialized_start=4025 + _globals['_K8SOBJECTMETADATA_LABELSENTRY']._serialized_end=4082 + _globals['_K8SOBJECTMETADATA_ANNOTATIONSENTRY']._serialized_start=4084 + _globals['_K8SOBJECTMETADATA_ANNOTATIONSENTRY']._serialized_end=4146 + _globals['_SQL']._serialized_start=4149 + _globals['_SQL']._serialized_end=4295 + _globals['_SQL_DIALECT']._serialized_start=4240 + _globals['_SQL_DIALECT']._serialized_end=4295 # @@protoc_insertion_point(module_scope) diff --git a/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.pyi b/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.pyi index 9e79c295ec..d7efcedbab 100644 --- a/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.pyi +++ b/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.pyi @@ -232,14 +232,16 @@ class DataLoadingConfig(_message.Message): def __init__(self, enabled: bool = ..., input_path: _Optional[str] = ..., output_path: _Optional[str] = ..., format: _Optional[_Union[DataLoadingConfig.LiteralMapFormat, str]] = ..., io_strategy: _Optional[_Union[IOStrategy, _Mapping]] = ...) -> None: ... class K8sPod(_message.Message): - __slots__ = ["metadata", "pod_spec", "data_config"] + __slots__ = ["metadata", "pod_spec", "data_config", "primarycontainername"] METADATA_FIELD_NUMBER: _ClassVar[int] POD_SPEC_FIELD_NUMBER: _ClassVar[int] DATA_CONFIG_FIELD_NUMBER: _ClassVar[int] + PRIMARYCONTAINERNAME_FIELD_NUMBER: _ClassVar[int] metadata: K8sObjectMetadata pod_spec: _struct_pb2.Struct data_config: DataLoadingConfig - def __init__(self, metadata: _Optional[_Union[K8sObjectMetadata, _Mapping]] = ..., pod_spec: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., data_config: _Optional[_Union[DataLoadingConfig, _Mapping]] = ...) -> None: ... + primarycontainername: str + def __init__(self, metadata: _Optional[_Union[K8sObjectMetadata, _Mapping]] = ..., pod_spec: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., data_config: _Optional[_Union[DataLoadingConfig, _Mapping]] = ..., primarycontainername: _Optional[str] = ...) -> None: ... class K8sObjectMetadata(_message.Message): __slots__ = ["labels", "annotations"] diff --git a/flyteidl/gen/pb_rust/flyteidl.core.rs b/flyteidl/gen/pb_rust/flyteidl.core.rs index 2d9ee08fe1..53d7184f68 100644 --- a/flyteidl/gen/pb_rust/flyteidl.core.rs +++ b/flyteidl/gen/pb_rust/flyteidl.core.rs @@ -1736,6 +1736,8 @@ pub struct K8sPod { /// Only K8s #[prost(message, optional, tag="3")] pub data_config: ::core::option::Option, + #[prost(string, tag="4")] + pub primarycontainername: ::prost::alloc::string::String, } /// Metadata for building a kubernetes object when a task is executed. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/flyteidl/protos/flyteidl/core/tasks.proto b/flyteidl/protos/flyteidl/core/tasks.proto index 332f9fdad4..24ca78a640 100644 --- a/flyteidl/protos/flyteidl/core/tasks.proto +++ b/flyteidl/protos/flyteidl/core/tasks.proto @@ -320,6 +320,8 @@ message K8sPod { // to understand the default paths. // Only K8s DataLoadingConfig data_config = 3; + + string primarycontainername = 4; } // Metadata for building a kubernetes object when a task is executed. diff --git a/flyteidl/setup.py b/flyteidl/setup.py index e633f16321..175b6d117b 100644 --- a/flyteidl/setup.py +++ b/flyteidl/setup.py @@ -1,3 +1,3 @@ from setuptools import setup, find_packages -setup(version='1.13.9') \ No newline at end of file +setup() \ No newline at end of file diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go index 6beca78f54..6b46ae0c3c 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -271,7 +271,6 @@ func BuildRawPod(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (*v Labels: make(map[string]string), } primaryContainerName := "" - switch target := taskTemplate.GetTarget().(type) { case *core.TaskTemplate_Container: // handles tasks defined by a single container @@ -286,6 +285,11 @@ func BuildRawPod(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (*v *c, }, } + if len(tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().Primarycontainername) > 0 { + podSpec, objectMeta, err = ApplyPodTemplateOverride(podSpec, objectMeta, tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) + primaryContainerName = tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().Primarycontainername + } + case *core.TaskTemplate_K8SPod: // handles pod tasks that marshal the pod spec to the k8s_pod task target. if target.K8SPod.GetPodSpec() == nil { @@ -337,6 +341,7 @@ func hasExternalLinkType(taskTemplate *core.TaskTemplate) bool { // configuration PodTemplate (if exists). func ApplyFlytePodConfiguration(ctx context.Context, tCtx pluginsCore.TaskExecutionContext, podSpec *v1.PodSpec, objectMeta *metav1.ObjectMeta, primaryContainerName string) (*v1.PodSpec, *metav1.ObjectMeta, error) { taskTemplate, err := tCtx.TaskReader().Read(ctx) + if err != nil { logger.Warnf(ctx, "failed to read task information when trying to construct Pod, err: %s", err.Error()) return nil, nil, err @@ -434,6 +439,13 @@ func ApplyFlytePodConfiguration(ctx context.Context, tCtx pluginsCore.TaskExecut ApplyContainerImageOverride(podSpec, tCtx.TaskExecutionMetadata().GetOverrides().GetContainerImage(), primaryContainerName) } + //if tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate() != nil { + // podSpec, objectMeta, err = ApplyPodTemplateOverride(podSpec, objectMeta, tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) + // if err != nil { + // return nil, nil, err + // } + //} + return podSpec, objectMeta, nil } @@ -446,6 +458,25 @@ func ApplyContainerImageOverride(podSpec *v1.PodSpec, containerImage string, pri } } +func ApplyPodTemplateOverride(podSpec *v1.PodSpec, objectMeta metav1.ObjectMeta, podtemplate *core.K8SPod) (*v1.PodSpec, metav1.ObjectMeta, error) { + if podtemplate.Metadata.Annotations != nil { + mergeMapInto(podtemplate.Metadata.Annotations, objectMeta.Annotations) + } + if podtemplate.Metadata.Labels != nil { + mergeMapInto(podtemplate.Metadata.Labels, objectMeta.Labels) + } + var podspec_override *v1.PodSpec + err := utils.UnmarshalStructToObj(podtemplate.PodSpec, &podspec_override) + if err != nil { + return nil, objectMeta, err + } + if podspec_override.Containers == nil { + return podSpec, objectMeta, nil + } else { + return podspec_override, objectMeta, nil + } +} + func addTolerationInPodSpec(podSpec *v1.PodSpec, toleration *v1.Toleration) *v1.PodSpec { podTolerations := podSpec.Tolerations From 01a131ba616235adfbba602fbeddecf38ca68322 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Fri, 20 Dec 2024 04:59:56 +0800 Subject: [PATCH 05/13] podtemplate override Signed-off-by: Nelson Chen --- .../go/tasks/pluginmachinery/flytek8s/pod_helper.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go index 6b46ae0c3c..7e4e59bcf1 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -439,13 +439,6 @@ func ApplyFlytePodConfiguration(ctx context.Context, tCtx pluginsCore.TaskExecut ApplyContainerImageOverride(podSpec, tCtx.TaskExecutionMetadata().GetOverrides().GetContainerImage(), primaryContainerName) } - //if tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate() != nil { - // podSpec, objectMeta, err = ApplyPodTemplateOverride(podSpec, objectMeta, tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) - // if err != nil { - // return nil, nil, err - // } - //} - return podSpec, objectMeta, nil } From c6d0a910ff9154a579636412e3e6d952dcdd5cf9 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Fri, 27 Dec 2024 23:03:43 +0800 Subject: [PATCH 06/13] add comment and test Signed-off-by: Nelson Chen --- flyteidl/clients/go/assets/admin.swagger.json | 3 +- flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts | 2 + flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go | 5 +- .../flyteidl/service/admin.swagger.json | 3 +- .../flyteidl/service/agent.swagger.json | 3 +- .../external_plugin_service.swagger.json | 3 +- flyteidl/gen/pb_rust/flyteidl.core.rs | 1 + flyteidl/protos/flyteidl/core/tasks.proto | 2 + .../pluginmachinery/flytek8s/pod_helper.go | 8 ++- .../flytek8s/pod_helper_test.go | 59 +++++++++++-------- 10 files changed, 57 insertions(+), 32 deletions(-) diff --git a/flyteidl/clients/go/assets/admin.swagger.json b/flyteidl/clients/go/assets/admin.swagger.json index f320a989c2..061af4263b 100644 --- a/flyteidl/clients/go/assets/admin.swagger.json +++ b/flyteidl/clients/go/assets/admin.swagger.json @@ -7379,7 +7379,8 @@ "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" }, "primarycontainername": { - "type": "string" + "type": "string", + "description": "Defines the primary container name when pod template override is executed." } }, "description": "Defines a pod spec and additional pod metadata that is created when a task is executed." diff --git a/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts b/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts index 6dbdac1521..e5c1160663 100644 --- a/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts +++ b/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts @@ -1102,6 +1102,8 @@ export class K8sPod extends Message { dataConfig?: DataLoadingConfig; /** + * Defines the primary container name when pod template override is executed. + * * @generated from field: string primarycontainername = 4; */ primarycontainername = ""; diff --git a/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go b/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go index cfea868473..5f02ca1f94 100644 --- a/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go @@ -1402,8 +1402,9 @@ type K8SPod struct { // are not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation // to understand the default paths. // Only K8s - DataConfig *DataLoadingConfig `protobuf:"bytes,3,opt,name=data_config,json=dataConfig,proto3" json:"data_config,omitempty"` - Primarycontainername string `protobuf:"bytes,4,opt,name=primarycontainername,proto3" json:"primarycontainername,omitempty"` + DataConfig *DataLoadingConfig `protobuf:"bytes,3,opt,name=data_config,json=dataConfig,proto3" json:"data_config,omitempty"` + // Defines the primary container name when pod template override is executed. + Primarycontainername string `protobuf:"bytes,4,opt,name=primarycontainername,proto3" json:"primarycontainername,omitempty"` } func (x *K8SPod) Reset() { diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json index f320a989c2..061af4263b 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json @@ -7379,7 +7379,8 @@ "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" }, "primarycontainername": { - "type": "string" + "type": "string", + "description": "Defines the primary container name when pod template override is executed." } }, "description": "Defines a pod spec and additional pod metadata that is created when a task is executed." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json index e725acfc45..4a3f1bdfb8 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json @@ -1258,7 +1258,8 @@ "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" }, "primarycontainername": { - "type": "string" + "type": "string", + "description": "Defines the primary container name when pod template override is executed." } }, "description": "Defines a pod spec and additional pod metadata that is created when a task is executed." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json index 65e2809e95..687de34ca7 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json @@ -533,7 +533,8 @@ "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" }, "primarycontainername": { - "type": "string" + "type": "string", + "description": "Defines the primary container name when pod template override is executed." } }, "description": "Defines a pod spec and additional pod metadata that is created when a task is executed." diff --git a/flyteidl/gen/pb_rust/flyteidl.core.rs b/flyteidl/gen/pb_rust/flyteidl.core.rs index 53d7184f68..5ba8b64819 100644 --- a/flyteidl/gen/pb_rust/flyteidl.core.rs +++ b/flyteidl/gen/pb_rust/flyteidl.core.rs @@ -1736,6 +1736,7 @@ pub struct K8sPod { /// Only K8s #[prost(message, optional, tag="3")] pub data_config: ::core::option::Option, + /// Defines the primary container name when pod template override is executed. #[prost(string, tag="4")] pub primarycontainername: ::prost::alloc::string::String, } diff --git a/flyteidl/protos/flyteidl/core/tasks.proto b/flyteidl/protos/flyteidl/core/tasks.proto index 24ca78a640..b0b7db5eaf 100644 --- a/flyteidl/protos/flyteidl/core/tasks.proto +++ b/flyteidl/protos/flyteidl/core/tasks.proto @@ -321,6 +321,8 @@ message K8sPod { // Only K8s DataLoadingConfig data_config = 3; + + // Defines the primary container name when pod template override is executed. string primarycontainername = 4; } diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go index 7e4e59bcf1..13564ba0bc 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -285,9 +285,11 @@ func BuildRawPod(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (*v *c, }, } - if len(tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().Primarycontainername) > 0 { - podSpec, objectMeta, err = ApplyPodTemplateOverride(podSpec, objectMeta, tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) - primaryContainerName = tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().Primarycontainername + if tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate() != nil { + if len(tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().Primarycontainername) > 0 { + podSpec, objectMeta, err = ApplyPodTemplateOverride(podSpec, objectMeta, tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) + primaryContainerName = tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().Primarycontainername + } } case *core.TaskTemplate_K8SPod: diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go index 0a70cdd895..b944a6bfe7 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go @@ -27,7 +27,7 @@ import ( "github.com/flyteorg/flyte/flytestdlib/storage" ) -func dummyTaskExecutionMetadata(resources *v1.ResourceRequirements, extendedResources *core.ExtendedResources, containerImage string) pluginsCore.TaskExecutionMetadata { +func dummyTaskExecutionMetadata(resources *v1.ResourceRequirements, extendedResources *core.ExtendedResources, containerImage string, podtemplate *core.K8SPod) pluginsCore.TaskExecutionMetadata { taskExecutionMetadata := &pluginsCoreMock.TaskExecutionMetadata{} taskExecutionMetadata.On("GetNamespace").Return("test-namespace") taskExecutionMetadata.On("GetAnnotations").Return(map[string]string{"annotation-1": "val1"}) @@ -54,6 +54,7 @@ func dummyTaskExecutionMetadata(resources *v1.ResourceRequirements, extendedReso to.On("GetResources").Return(resources) to.On("GetExtendedResources").Return(extendedResources) to.On("GetContainerImage").Return(containerImage) + to.On("GetPodTemplate").Return(podtemplate) taskExecutionMetadata.On("GetOverrides").Return(to) taskExecutionMetadata.On("IsInterruptible").Return(true) taskExecutionMetadata.OnGetPlatformResources().Return(&v1.ResourceRequirements{}) @@ -82,7 +83,7 @@ func dummyInputReader() io.InputReader { return inputReader } -func dummyExecContext(taskTemplate *core.TaskTemplate, r *v1.ResourceRequirements, rm *core.ExtendedResources, containerImage string) pluginsCore.TaskExecutionContext { +func dummyExecContext(taskTemplate *core.TaskTemplate, r *v1.ResourceRequirements, rm *core.ExtendedResources, containerImage string, podtemplate *core.K8SPod) pluginsCore.TaskExecutionContext { ow := &pluginsIOMock.OutputWriter{} ow.OnGetOutputPrefixPath().Return("") ow.OnGetRawOutputPrefix().Return("") @@ -90,7 +91,7 @@ func dummyExecContext(taskTemplate *core.TaskTemplate, r *v1.ResourceRequirement ow.OnGetPreviousCheckpointsPrefix().Return("/prev") tCtx := &pluginsCoreMock.TaskExecutionContext{} - tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(r, rm, containerImage)) + tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(r, rm, containerImage, podtemplate)) tCtx.OnInputReader().Return(dummyInputReader()) tCtx.OnOutputWriter().Return(ow) @@ -703,7 +704,7 @@ func updatePod(t *testing.T) { v1.ResourceCPU: resource.MustParse("1024m"), v1.ResourceEphemeralStorage: resource.MustParse("100M"), }, - }, nil, "") + }, nil, "", nil) pod := &v1.Pod{ Spec: v1.PodSpec{ @@ -756,7 +757,7 @@ func updatePod(t *testing.T) { } func TestUpdatePodWithDefaultAffinityAndInterruptibleNodeSelectorRequirement(t *testing.T) { - taskExecutionMetadata := dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "") + taskExecutionMetadata := dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "", nil) assert.NoError(t, config.SetK8sPluginConfig(&config.K8sPluginConfig{ DefaultAffinity: &v1.Affinity{ NodeAffinity: &v1.NodeAffinity{ @@ -820,7 +821,7 @@ func toK8sPodInterruptible(t *testing.T) { v1.ResourceCPU: resource.MustParse("1024m"), v1.ResourceEphemeralStorage: resource.MustParse("100M"), }, - }, nil, "") + }, nil, "", nil) p, _, _, err := ToK8sPodSpec(ctx, x) assert.NoError(t, err) @@ -887,7 +888,7 @@ func TestToK8sPod(t *testing.T) { v1.ResourceCPU: resource.MustParse("1024m"), v1.ResourceEphemeralStorage: resource.MustParse("100M"), }, - }, nil, "") + }, nil, "", nil) p, _, _, err := ToK8sPodSpec(ctx, x) assert.NoError(t, err) @@ -904,7 +905,7 @@ func TestToK8sPod(t *testing.T) { v1.ResourceCPU: resource.MustParse("1024m"), v1.ResourceEphemeralStorage: resource.MustParse("100M"), }, - }, nil, "") + }, nil, "", nil) p, _, _, err := ToK8sPodSpec(ctx, x) assert.NoError(t, err) @@ -922,7 +923,7 @@ func TestToK8sPod(t *testing.T) { v1.ResourceCPU: resource.MustParse("1024m"), v1.ResourceEphemeralStorage: resource.MustParse("100M"), }, - }, nil, "") + }, nil, "", nil) assert.NoError(t, config.SetK8sPluginConfig(&config.K8sPluginConfig{ DefaultNodeSelector: map[string]string{ @@ -949,7 +950,7 @@ func TestToK8sPod(t *testing.T) { }, })) - x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "") + x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "", nil) p, _, _, err := ToK8sPodSpec(ctx, x) assert.NoError(t, err) assert.NotNil(t, p.SecurityContext) @@ -961,7 +962,7 @@ func TestToK8sPod(t *testing.T) { assert.NoError(t, config.SetK8sPluginConfig(&config.K8sPluginConfig{ EnableHostNetworkingPod: &enabled, })) - x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "") + x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "", nil) p, _, _, err := ToK8sPodSpec(ctx, x) assert.NoError(t, err) assert.True(t, p.HostNetwork) @@ -972,7 +973,7 @@ func TestToK8sPod(t *testing.T) { assert.NoError(t, config.SetK8sPluginConfig(&config.K8sPluginConfig{ EnableHostNetworkingPod: &enabled, })) - x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "") + x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "", nil) p, _, _, err := ToK8sPodSpec(ctx, x) assert.NoError(t, err) assert.False(t, p.HostNetwork) @@ -980,7 +981,7 @@ func TestToK8sPod(t *testing.T) { t.Run("skipSettingHostNetwork", func(t *testing.T) { assert.NoError(t, config.SetK8sPluginConfig(&config.K8sPluginConfig{})) - x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "") + x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "", nil) p, _, _, err := ToK8sPodSpec(ctx, x) assert.NoError(t, err) assert.False(t, p.HostNetwork) @@ -1014,7 +1015,7 @@ func TestToK8sPod(t *testing.T) { }, })) - x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "") + x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "", nil) p, _, _, err := ToK8sPodSpec(ctx, x) assert.NoError(t, err) assert.NotNil(t, p.DNSConfig) @@ -1035,7 +1036,7 @@ func TestToK8sPod(t *testing.T) { "foo": "bar", }, })) - x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "") + x := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{}, nil, "", nil) p, _, _, err := ToK8sPodSpec(ctx, x) assert.NoError(t, err) for _, c := range p.Containers { @@ -1055,7 +1056,19 @@ func TestToK8sPodContainerImage(t *testing.T) { taskContext := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{ Requests: v1.ResourceList{ v1.ResourceCPU: resource.MustParse("1024m"), - }}, nil, "foo:latest") + }}, nil, "foo:latest", nil) + p, _, _, err := ToK8sPodSpec(context.TODO(), taskContext) + assert.NoError(t, err) + assert.Equal(t, "foo:latest", p.Containers[0].Image) + }) +} + +func TestToK8sPodPodTemplate(t *testing.T) { + t.Run("Override pod template", func(t *testing.T) { + taskContext := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1024m"), + }}, nil, "", nil) p, _, _, err := ToK8sPodSpec(context.TODO(), taskContext) assert.NoError(t, err) assert.Equal(t, "foo:latest", p.Containers[0].Image) @@ -1167,7 +1180,7 @@ func TestToK8sPodExtendedResources(t *testing.T) { t.Run(f.name, func(t *testing.T) { taskTemplate := dummyTaskTemplate() taskTemplate.ExtendedResources = f.extendedResourcesBase - taskContext := dummyExecContext(taskTemplate, f.resources, f.extendedResourcesOverride, "") + taskContext := dummyExecContext(taskTemplate, f.resources, f.extendedResourcesOverride, "", nil) p, _, _, err := ToK8sPodSpec(context.TODO(), taskContext) assert.NoError(t, err) @@ -1829,7 +1842,7 @@ func TestGetPodTemplate(t *testing.T) { taskReader.On("Read", mock.Anything).Return(task, nil) tCtx := &pluginsCoreMock.TaskExecutionContext{} - tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "")) + tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "", nil)) tCtx.OnTaskReader().Return(taskReader) // initialize PodTemplateStore @@ -1855,7 +1868,7 @@ func TestGetPodTemplate(t *testing.T) { taskReader.On("Read", mock.Anything).Return(task, nil) tCtx := &pluginsCoreMock.TaskExecutionContext{} - tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "")) + tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "", nil)) tCtx.OnTaskReader().Return(taskReader) // initialize PodTemplateStore @@ -1882,7 +1895,7 @@ func TestGetPodTemplate(t *testing.T) { taskReader.On("Read", mock.Anything).Return(task, nil) tCtx := &pluginsCoreMock.TaskExecutionContext{} - tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "")) + tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "", nil)) tCtx.OnTaskReader().Return(taskReader) // initialize PodTemplateStore @@ -1910,7 +1923,7 @@ func TestGetPodTemplate(t *testing.T) { taskReader.On("Read", mock.Anything).Return(task, nil) tCtx := &pluginsCoreMock.TaskExecutionContext{} - tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "")) + tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "", nil)) tCtx.OnTaskReader().Return(taskReader) // initialize PodTemplateStore @@ -1960,7 +1973,7 @@ func TestMergeWithBasePodTemplate(t *testing.T) { taskReader.On("Read", mock.Anything).Return(task, nil) tCtx := &pluginsCoreMock.TaskExecutionContext{} - tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "")) + tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "", nil)) tCtx.OnTaskReader().Return(taskReader) resultPodSpec, resultObjectMeta, err := MergeWithBasePodTemplate(context.TODO(), tCtx, &podSpec, &objectMeta, "foo", "foo-init") @@ -2022,7 +2035,7 @@ func TestMergeWithBasePodTemplate(t *testing.T) { taskReader.On("Read", mock.Anything).Return(task, nil) tCtx := &pluginsCoreMock.TaskExecutionContext{} - tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "")) + tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(&v1.ResourceRequirements{}, nil, "", nil)) tCtx.OnTaskReader().Return(taskReader) resultPodSpec, resultObjectMeta, err := MergeWithBasePodTemplate(context.TODO(), tCtx, &podSpec, &objectMeta, "foo", "foo-init") From 788e7d7131598fa93c04d474e53d1727bc93a089 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 28 Dec 2024 00:09:36 +0800 Subject: [PATCH 07/13] test Signed-off-by: Nelson Chen --- .../pluginmachinery/flytek8s/pod_helper_test.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go index b944a6bfe7..9b2f99f9fb 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go @@ -1063,18 +1063,6 @@ func TestToK8sPodContainerImage(t *testing.T) { }) } -func TestToK8sPodPodTemplate(t *testing.T) { - t.Run("Override pod template", func(t *testing.T) { - taskContext := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1024m"), - }}, nil, "", nil) - p, _, _, err := ToK8sPodSpec(context.TODO(), taskContext) - assert.NoError(t, err) - assert.Equal(t, "foo:latest", p.Containers[0].Image) - }) -} - func TestToK8sPodExtendedResources(t *testing.T) { assert.NoError(t, config.SetK8sPluginConfig(&config.K8sPluginConfig{ GpuDeviceNodeLabel: "gpu-node-label", From c15830de0c7724a770ac58e2133595bc82675e4f Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sun, 29 Dec 2024 23:37:32 +0800 Subject: [PATCH 08/13] fix typo Signed-off-by: Nelson Chen --- flyteidl/clients/go/assets/admin.swagger.json | 2 +- flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts | 6 +- flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go | 96 +++++++++---------- .../flyteidl/service/admin.swagger.json | 2 +- .../flyteidl/service/agent.swagger.json | 2 +- .../external_plugin_service.swagger.json | 2 +- flyteidl/gen/pb-js/flyteidl.d.ts | 8 +- flyteidl/gen/pb-js/flyteidl.js | 20 ++-- .../gen/pb_python/flyteidl/core/tasks_pb2.py | 24 ++--- .../gen/pb_python/flyteidl/core/tasks_pb2.pyi | 8 +- flyteidl/gen/pb_rust/flyteidl.core.rs | 2 +- flyteidl/protos/flyteidl/core/tasks.proto | 2 +- .../pluginmachinery/flytek8s/pod_helper.go | 6 +- 13 files changed, 90 insertions(+), 90 deletions(-) diff --git a/flyteidl/clients/go/assets/admin.swagger.json b/flyteidl/clients/go/assets/admin.swagger.json index 061af4263b..930a1cbe51 100644 --- a/flyteidl/clients/go/assets/admin.swagger.json +++ b/flyteidl/clients/go/assets/admin.swagger.json @@ -7378,7 +7378,7 @@ "$ref": "#/definitions/coreDataLoadingConfig", "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" }, - "primarycontainername": { + "primary_container_name": { "type": "string", "description": "Defines the primary container name when pod template override is executed." } diff --git a/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts b/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts index e5c1160663..1d06e1ac3e 100644 --- a/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts +++ b/flyteidl/gen/pb-es/flyteidl/core/tasks_pb.ts @@ -1104,9 +1104,9 @@ export class K8sPod extends Message { /** * Defines the primary container name when pod template override is executed. * - * @generated from field: string primarycontainername = 4; + * @generated from field: string primary_container_name = 4; */ - primarycontainername = ""; + primaryContainerName = ""; constructor(data?: PartialMessage) { super(); @@ -1119,7 +1119,7 @@ export class K8sPod extends Message { { no: 1, name: "metadata", kind: "message", T: K8sObjectMetadata }, { no: 2, name: "pod_spec", kind: "message", T: Struct }, { no: 3, name: "data_config", kind: "message", T: DataLoadingConfig }, - { no: 4, name: "primarycontainername", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 4, name: "primary_container_name", kind: "scalar", T: 9 /* ScalarType.STRING */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): K8sPod { diff --git a/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go b/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go index 5f02ca1f94..50a0e9bc3a 100644 --- a/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/core/tasks.pb.go @@ -1404,7 +1404,7 @@ type K8SPod struct { // Only K8s DataConfig *DataLoadingConfig `protobuf:"bytes,3,opt,name=data_config,json=dataConfig,proto3" json:"data_config,omitempty"` // Defines the primary container name when pod template override is executed. - Primarycontainername string `protobuf:"bytes,4,opt,name=primarycontainername,proto3" json:"primarycontainername,omitempty"` + PrimaryContainerName string `protobuf:"bytes,4,opt,name=primary_container_name,json=primaryContainerName,proto3" json:"primary_container_name,omitempty"` } func (x *K8SPod) Reset() { @@ -1460,9 +1460,9 @@ func (x *K8SPod) GetDataConfig() *DataLoadingConfig { return nil } -func (x *K8SPod) GetPrimarycontainername() string { +func (x *K8SPod) GetPrimaryContainerName() string { if x != nil { - return x.Primarycontainername + return x.PrimaryContainerName } return "" } @@ -1875,7 +1875,7 @@ var file_flyteidl_core_tasks_proto_rawDesc = []byte{ 0x31, 0x0a, 0x10, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x61, 0x70, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x08, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x59, 0x41, 0x4d, 0x4c, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x52, 0x4f, 0x54, 0x4f, - 0x10, 0x02, 0x22, 0xf1, 0x01, 0x0a, 0x06, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x12, 0x3c, 0x0a, + 0x10, 0x02, 0x22, 0xf3, 0x01, 0x0a, 0x06, 0x4b, 0x38, 0x73, 0x50, 0x6f, 0x64, 0x12, 0x3c, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4b, 0x38, 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, @@ -1887,50 +1887,50 @@ var file_flyteidl_core_tasks_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x4c, 0x6f, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x32, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xa9, 0x02, 0x0a, 0x11, 0x4b, 0x38, 0x73, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x44, 0x0a, 0x06, - 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x66, - 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4b, 0x38, 0x73, - 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, - 0x6c, 0x73, 0x12, 0x53, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, - 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4b, 0x38, 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x22, 0x92, 0x01, 0x0a, 0x03, 0x53, 0x71, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x64, 0x69, 0x61, 0x6c, - 0x65, 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x66, 0x6c, 0x79, 0x74, - 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x71, 0x6c, 0x2e, 0x44, 0x69, - 0x61, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x07, 0x64, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x22, 0x37, - 0x0a, 0x07, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x4e, 0x44, - 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4e, 0x53, 0x49, - 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, - 0x4f, 0x54, 0x48, 0x45, 0x52, 0x10, 0x03, 0x42, 0xb0, 0x01, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x2e, - 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0a, 0x54, - 0x61, 0x73, 0x6b, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3a, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x6f, 0x72, 0x67, - 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, - 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x62, 0x2d, 0x67, 0x6f, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, - 0x64, 0x6c, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0xa2, 0x02, 0x03, 0x46, 0x43, 0x58, 0xaa, 0x02, 0x0d, - 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0xca, 0x02, 0x0d, - 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0xe2, 0x02, 0x19, - 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0x5c, 0x47, 0x50, - 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0e, 0x46, 0x6c, 0x79, 0x74, - 0x65, 0x69, 0x64, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x69, 0x67, 0x12, 0x34, 0x0a, 0x16, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x14, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xa9, 0x02, 0x0a, 0x11, 0x4b, 0x38, 0x73, + 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x44, + 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, + 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4b, + 0x38, 0x73, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, + 0x62, 0x65, 0x6c, 0x73, 0x12, 0x53, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x66, 0x6c, 0x79, 0x74, + 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4b, 0x38, 0x73, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, + 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, + 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, + 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x92, 0x01, 0x0a, 0x03, 0x53, 0x71, 0x6c, 0x12, 0x1c, 0x0a, 0x09, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x64, 0x69, + 0x61, 0x6c, 0x65, 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x66, 0x6c, + 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x71, 0x6c, 0x2e, + 0x44, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x07, 0x64, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, + 0x22, 0x37, 0x0a, 0x07, 0x44, 0x69, 0x61, 0x6c, 0x65, 0x63, 0x74, 0x12, 0x0d, 0x0a, 0x09, 0x55, + 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4e, + 0x53, 0x49, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x09, + 0x0a, 0x05, 0x4f, 0x54, 0x48, 0x45, 0x52, 0x10, 0x03, 0x42, 0xb0, 0x01, 0x0a, 0x11, 0x63, 0x6f, + 0x6d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, + 0x0a, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3a, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x6f, + 0x72, 0x67, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, + 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x62, 0x2d, 0x67, 0x6f, 0x2f, 0x66, 0x6c, 0x79, 0x74, + 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0xa2, 0x02, 0x03, 0x46, 0x43, 0x58, 0xaa, + 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0xca, + 0x02, 0x0d, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0xe2, + 0x02, 0x19, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x43, 0x6f, 0x72, 0x65, 0x5c, + 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0e, 0x46, 0x6c, + 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x3a, 0x3a, 0x43, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json index 061af4263b..930a1cbe51 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json @@ -7378,7 +7378,7 @@ "$ref": "#/definitions/coreDataLoadingConfig", "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" }, - "primarycontainername": { + "primary_container_name": { "type": "string", "description": "Defines the primary container name when pod template override is executed." } diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json index 4a3f1bdfb8..2207225195 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json @@ -1257,7 +1257,7 @@ "$ref": "#/definitions/coreDataLoadingConfig", "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" }, - "primarycontainername": { + "primary_container_name": { "type": "string", "description": "Defines the primary container name when pod template override is executed." } diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json index 687de34ca7..eacc2cfeab 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/external_plugin_service.swagger.json @@ -532,7 +532,7 @@ "$ref": "#/definitions/coreDataLoadingConfig", "title": "BETA: Optional configuration for DataLoading. If not specified, then default values are used.\nThis makes it possible to to run a completely portable container, that uses inputs and outputs\nonly from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.\nIf data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories\nare not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation\nto understand the default paths.\nOnly K8s" }, - "primarycontainername": { + "primary_container_name": { "type": "string", "description": "Defines the primary container name when pod template override is executed." } diff --git a/flyteidl/gen/pb-js/flyteidl.d.ts b/flyteidl/gen/pb-js/flyteidl.d.ts index 9e73fb7a2d..95c8eaccfa 100644 --- a/flyteidl/gen/pb-js/flyteidl.d.ts +++ b/flyteidl/gen/pb-js/flyteidl.d.ts @@ -6960,8 +6960,8 @@ export namespace flyteidl { /** K8sPod dataConfig */ dataConfig?: (flyteidl.core.IDataLoadingConfig|null); - /** K8sPod primarycontainername */ - primarycontainername?: (string|null); + /** K8sPod primaryContainerName */ + primaryContainerName?: (string|null); } /** Represents a K8sPod. */ @@ -6982,8 +6982,8 @@ export namespace flyteidl { /** K8sPod dataConfig. */ public dataConfig?: (flyteidl.core.IDataLoadingConfig|null); - /** K8sPod primarycontainername. */ - public primarycontainername: string; + /** K8sPod primaryContainerName. */ + public primaryContainerName: string; /** * Creates a new K8sPod instance using the specified properties. diff --git a/flyteidl/gen/pb-js/flyteidl.js b/flyteidl/gen/pb-js/flyteidl.js index b931baea2b..a90a9c15d5 100644 --- a/flyteidl/gen/pb-js/flyteidl.js +++ b/flyteidl/gen/pb-js/flyteidl.js @@ -16850,7 +16850,7 @@ * @property {flyteidl.core.IK8sObjectMetadata|null} [metadata] K8sPod metadata * @property {google.protobuf.IStruct|null} [podSpec] K8sPod podSpec * @property {flyteidl.core.IDataLoadingConfig|null} [dataConfig] K8sPod dataConfig - * @property {string|null} [primarycontainername] K8sPod primarycontainername + * @property {string|null} [primaryContainerName] K8sPod primaryContainerName */ /** @@ -16893,12 +16893,12 @@ K8sPod.prototype.dataConfig = null; /** - * K8sPod primarycontainername. - * @member {string} primarycontainername + * K8sPod primaryContainerName. + * @member {string} primaryContainerName * @memberof flyteidl.core.K8sPod * @instance */ - K8sPod.prototype.primarycontainername = ""; + K8sPod.prototype.primaryContainerName = ""; /** * Creates a new K8sPod instance using the specified properties. @@ -16930,8 +16930,8 @@ $root.google.protobuf.Struct.encode(message.podSpec, writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim(); if (message.dataConfig != null && message.hasOwnProperty("dataConfig")) $root.flyteidl.core.DataLoadingConfig.encode(message.dataConfig, writer.uint32(/* id 3, wireType 2 =*/26).fork()).ldelim(); - if (message.primarycontainername != null && message.hasOwnProperty("primarycontainername")) - writer.uint32(/* id 4, wireType 2 =*/34).string(message.primarycontainername); + if (message.primaryContainerName != null && message.hasOwnProperty("primaryContainerName")) + writer.uint32(/* id 4, wireType 2 =*/34).string(message.primaryContainerName); return writer; }; @@ -16963,7 +16963,7 @@ message.dataConfig = $root.flyteidl.core.DataLoadingConfig.decode(reader, reader.uint32()); break; case 4: - message.primarycontainername = reader.string(); + message.primaryContainerName = reader.string(); break; default: reader.skipType(tag & 7); @@ -16999,9 +16999,9 @@ if (error) return "dataConfig." + error; } - if (message.primarycontainername != null && message.hasOwnProperty("primarycontainername")) - if (!$util.isString(message.primarycontainername)) - return "primarycontainername: string expected"; + if (message.primaryContainerName != null && message.hasOwnProperty("primaryContainerName")) + if (!$util.isString(message.primaryContainerName)) + return "primaryContainerName: string expected"; return null; }; diff --git a/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.py b/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.py index 762128263b..1f3aa78714 100644 --- a/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.py +++ b/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.py @@ -19,7 +19,7 @@ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19\x66lyteidl/core/tasks.proto\x12\rflyteidl.core\x1a\x1e\x66lyteidl/core/identifier.proto\x1a\x1d\x66lyteidl/core/interface.proto\x1a\x1c\x66lyteidl/core/literals.proto\x1a\x1c\x66lyteidl/core/security.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xd0\x02\n\tResources\x12\x42\n\x08requests\x18\x01 \x03(\x0b\x32&.flyteidl.core.Resources.ResourceEntryR\x08requests\x12>\n\x06limits\x18\x02 \x03(\x0b\x32&.flyteidl.core.Resources.ResourceEntryR\x06limits\x1a`\n\rResourceEntry\x12\x39\n\x04name\x18\x01 \x01(\x0e\x32%.flyteidl.core.Resources.ResourceNameR\x04name\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\"]\n\x0cResourceName\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x07\n\x03\x43PU\x10\x01\x12\x07\n\x03GPU\x10\x02\x12\n\n\x06MEMORY\x10\x03\x12\x0b\n\x07STORAGE\x10\x04\x12\x15\n\x11\x45PHEMERAL_STORAGE\x10\x05\"\x91\x01\n\x0eGPUAccelerator\x12\x16\n\x06\x64\x65vice\x18\x01 \x01(\tR\x06\x64\x65vice\x12&\n\runpartitioned\x18\x02 \x01(\x08H\x00R\runpartitioned\x12\'\n\x0epartition_size\x18\x03 \x01(\tH\x00R\rpartitionSizeB\x16\n\x14partition_size_value\"[\n\x11\x45xtendedResources\x12\x46\n\x0fgpu_accelerator\x18\x01 \x01(\x0b\x32\x1d.flyteidl.core.GPUAcceleratorR\x0egpuAccelerator\"\xac\x01\n\x0fRuntimeMetadata\x12>\n\x04type\x18\x01 \x01(\x0e\x32*.flyteidl.core.RuntimeMetadata.RuntimeTypeR\x04type\x12\x18\n\x07version\x18\x02 \x01(\tR\x07version\x12\x16\n\x06\x66lavor\x18\x03 \x01(\tR\x06\x66lavor\"\'\n\x0bRuntimeType\x12\t\n\x05OTHER\x10\x00\x12\r\n\tFLYTE_SDK\x10\x01\"\xc7\x05\n\x0cTaskMetadata\x12\"\n\x0c\x64iscoverable\x18\x01 \x01(\x08R\x0c\x64iscoverable\x12\x38\n\x07runtime\x18\x02 \x01(\x0b\x32\x1e.flyteidl.core.RuntimeMetadataR\x07runtime\x12\x33\n\x07timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\x07timeout\x12\x36\n\x07retries\x18\x05 \x01(\x0b\x32\x1c.flyteidl.core.RetryStrategyR\x07retries\x12+\n\x11\x64iscovery_version\x18\x06 \x01(\tR\x10\x64iscoveryVersion\x12\x38\n\x18\x64\x65precated_error_message\x18\x07 \x01(\tR\x16\x64\x65precatedErrorMessage\x12&\n\rinterruptible\x18\x08 \x01(\x08H\x00R\rinterruptible\x12-\n\x12\x63\x61\x63he_serializable\x18\t \x01(\x08R\x11\x63\x61\x63heSerializable\x12%\n\x0egenerates_deck\x18\n \x01(\x08R\rgeneratesDeck\x12\x39\n\x04tags\x18\x0b \x03(\x0b\x32%.flyteidl.core.TaskMetadata.TagsEntryR\x04tags\x12*\n\x11pod_template_name\x18\x0c \x01(\tR\x0fpodTemplateName\x12\x35\n\x17\x63\x61\x63he_ignore_input_vars\x18\r \x03(\tR\x14\x63\x61\x63heIgnoreInputVars\x12\x19\n\x08is_eager\x18\x0e \x01(\x08R\x07isEager\x1a\x37\n\tTagsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x15\n\x13interruptible_value\"\xd6\x05\n\x0cTaskTemplate\x12)\n\x02id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierR\x02id\x12\x12\n\x04type\x18\x02 \x01(\tR\x04type\x12\x37\n\x08metadata\x18\x03 \x01(\x0b\x32\x1b.flyteidl.core.TaskMetadataR\x08metadata\x12;\n\tinterface\x18\x04 \x01(\x0b\x32\x1d.flyteidl.core.TypedInterfaceR\tinterface\x12/\n\x06\x63ustom\x18\x05 \x01(\x0b\x32\x17.google.protobuf.StructR\x06\x63ustom\x12\x38\n\tcontainer\x18\x06 \x01(\x0b\x32\x18.flyteidl.core.ContainerH\x00R\tcontainer\x12\x30\n\x07k8s_pod\x18\x11 \x01(\x0b\x32\x15.flyteidl.core.K8sPodH\x00R\x06k8sPod\x12&\n\x03sql\x18\x12 \x01(\x0b\x32\x12.flyteidl.core.SqlH\x00R\x03sql\x12*\n\x11task_type_version\x18\x07 \x01(\x05R\x0ftaskTypeVersion\x12I\n\x10security_context\x18\x08 \x01(\x0b\x32\x1e.flyteidl.core.SecurityContextR\x0fsecurityContext\x12O\n\x12\x65xtended_resources\x18\t \x01(\x0b\x32 .flyteidl.core.ExtendedResourcesR\x11\x65xtendedResources\x12?\n\x06\x63onfig\x18\x10 \x03(\x0b\x32\'.flyteidl.core.TaskTemplate.ConfigEntryR\x06\x63onfig\x1a\x39\n\x0b\x43onfigEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x08\n\x06target\"6\n\rContainerPort\x12%\n\x0e\x63ontainer_port\x18\x01 \x01(\rR\rcontainerPort\"\xfc\x03\n\tContainer\x12\x14\n\x05image\x18\x01 \x01(\tR\x05image\x12\x18\n\x07\x63ommand\x18\x02 \x03(\tR\x07\x63ommand\x12\x12\n\x04\x61rgs\x18\x03 \x03(\tR\x04\x61rgs\x12\x36\n\tresources\x18\x04 \x01(\x0b\x32\x18.flyteidl.core.ResourcesR\tresources\x12-\n\x03\x65nv\x18\x05 \x03(\x0b\x32\x1b.flyteidl.core.KeyValuePairR\x03\x65nv\x12\x37\n\x06\x63onfig\x18\x06 \x03(\x0b\x32\x1b.flyteidl.core.KeyValuePairB\x02\x18\x01R\x06\x63onfig\x12\x32\n\x05ports\x18\x07 \x03(\x0b\x32\x1c.flyteidl.core.ContainerPortR\x05ports\x12\x41\n\x0b\x64\x61ta_config\x18\t \x01(\x0b\x32 .flyteidl.core.DataLoadingConfigR\ndataConfig\x12I\n\x0c\x61rchitecture\x18\n \x01(\x0e\x32%.flyteidl.core.Container.ArchitectureR\x0c\x61rchitecture\"I\n\x0c\x41rchitecture\x12\x0b\n\x07UNKNOWN\x10\x00\x12\t\n\x05\x41MD64\x10\x01\x12\t\n\x05\x41RM64\x10\x02\x12\n\n\x06\x41RM_V6\x10\x03\x12\n\n\x06\x41RM_V7\x10\x04\"\xb5\x02\n\nIOStrategy\x12K\n\rdownload_mode\x18\x01 \x01(\x0e\x32&.flyteidl.core.IOStrategy.DownloadModeR\x0c\x64ownloadMode\x12\x45\n\x0bupload_mode\x18\x02 \x01(\x0e\x32$.flyteidl.core.IOStrategy.UploadModeR\nuploadMode\"L\n\x0c\x44ownloadMode\x12\x12\n\x0e\x44OWNLOAD_EAGER\x10\x00\x12\x13\n\x0f\x44OWNLOAD_STREAM\x10\x01\x12\x13\n\x0f\x44O_NOT_DOWNLOAD\x10\x02\"E\n\nUploadMode\x12\x12\n\x0eUPLOAD_ON_EXIT\x10\x00\x12\x10\n\x0cUPLOAD_EAGER\x10\x01\x12\x11\n\rDO_NOT_UPLOAD\x10\x02\"\xa7\x02\n\x11\x44\x61taLoadingConfig\x12\x18\n\x07\x65nabled\x18\x01 \x01(\x08R\x07\x65nabled\x12\x1d\n\ninput_path\x18\x02 \x01(\tR\tinputPath\x12\x1f\n\x0boutput_path\x18\x03 \x01(\tR\noutputPath\x12I\n\x06\x66ormat\x18\x04 \x01(\x0e\x32\x31.flyteidl.core.DataLoadingConfig.LiteralMapFormatR\x06\x66ormat\x12:\n\x0bio_strategy\x18\x05 \x01(\x0b\x32\x19.flyteidl.core.IOStrategyR\nioStrategy\"1\n\x10LiteralMapFormat\x12\x08\n\x04JSON\x10\x00\x12\x08\n\x04YAML\x10\x01\x12\t\n\x05PROTO\x10\x02\"\xf1\x01\n\x06K8sPod\x12<\n\x08metadata\x18\x01 \x01(\x0b\x32 .flyteidl.core.K8sObjectMetadataR\x08metadata\x12\x32\n\x08pod_spec\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x07podSpec\x12\x41\n\x0b\x64\x61ta_config\x18\x03 \x01(\x0b\x32 .flyteidl.core.DataLoadingConfigR\ndataConfig\x12\x32\n\x14primarycontainername\x18\x04 \x01(\tR\x14primarycontainername\"\xa9\x02\n\x11K8sObjectMetadata\x12\x44\n\x06labels\x18\x01 \x03(\x0b\x32,.flyteidl.core.K8sObjectMetadata.LabelsEntryR\x06labels\x12S\n\x0b\x61nnotations\x18\x02 \x03(\x0b\x32\x31.flyteidl.core.K8sObjectMetadata.AnnotationsEntryR\x0b\x61nnotations\x1a\x39\n\x0bLabelsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a>\n\x10\x41nnotationsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"\x92\x01\n\x03Sql\x12\x1c\n\tstatement\x18\x01 \x01(\tR\tstatement\x12\x34\n\x07\x64ialect\x18\x02 \x01(\x0e\x32\x1a.flyteidl.core.Sql.DialectR\x07\x64ialect\"7\n\x07\x44ialect\x12\r\n\tUNDEFINED\x10\x00\x12\x08\n\x04\x41NSI\x10\x01\x12\x08\n\x04HIVE\x10\x02\x12\t\n\x05OTHER\x10\x03\x42\xb0\x01\n\x11\x63om.flyteidl.coreB\nTasksProtoP\x01Z:github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core\xa2\x02\x03\x46\x43X\xaa\x02\rFlyteidl.Core\xca\x02\rFlyteidl\\Core\xe2\x02\x19\x46lyteidl\\Core\\GPBMetadata\xea\x02\x0e\x46lyteidl::Coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19\x66lyteidl/core/tasks.proto\x12\rflyteidl.core\x1a\x1e\x66lyteidl/core/identifier.proto\x1a\x1d\x66lyteidl/core/interface.proto\x1a\x1c\x66lyteidl/core/literals.proto\x1a\x1c\x66lyteidl/core/security.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xd0\x02\n\tResources\x12\x42\n\x08requests\x18\x01 \x03(\x0b\x32&.flyteidl.core.Resources.ResourceEntryR\x08requests\x12>\n\x06limits\x18\x02 \x03(\x0b\x32&.flyteidl.core.Resources.ResourceEntryR\x06limits\x1a`\n\rResourceEntry\x12\x39\n\x04name\x18\x01 \x01(\x0e\x32%.flyteidl.core.Resources.ResourceNameR\x04name\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\"]\n\x0cResourceName\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x07\n\x03\x43PU\x10\x01\x12\x07\n\x03GPU\x10\x02\x12\n\n\x06MEMORY\x10\x03\x12\x0b\n\x07STORAGE\x10\x04\x12\x15\n\x11\x45PHEMERAL_STORAGE\x10\x05\"\x91\x01\n\x0eGPUAccelerator\x12\x16\n\x06\x64\x65vice\x18\x01 \x01(\tR\x06\x64\x65vice\x12&\n\runpartitioned\x18\x02 \x01(\x08H\x00R\runpartitioned\x12\'\n\x0epartition_size\x18\x03 \x01(\tH\x00R\rpartitionSizeB\x16\n\x14partition_size_value\"[\n\x11\x45xtendedResources\x12\x46\n\x0fgpu_accelerator\x18\x01 \x01(\x0b\x32\x1d.flyteidl.core.GPUAcceleratorR\x0egpuAccelerator\"\xac\x01\n\x0fRuntimeMetadata\x12>\n\x04type\x18\x01 \x01(\x0e\x32*.flyteidl.core.RuntimeMetadata.RuntimeTypeR\x04type\x12\x18\n\x07version\x18\x02 \x01(\tR\x07version\x12\x16\n\x06\x66lavor\x18\x03 \x01(\tR\x06\x66lavor\"\'\n\x0bRuntimeType\x12\t\n\x05OTHER\x10\x00\x12\r\n\tFLYTE_SDK\x10\x01\"\xc7\x05\n\x0cTaskMetadata\x12\"\n\x0c\x64iscoverable\x18\x01 \x01(\x08R\x0c\x64iscoverable\x12\x38\n\x07runtime\x18\x02 \x01(\x0b\x32\x1e.flyteidl.core.RuntimeMetadataR\x07runtime\x12\x33\n\x07timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\x07timeout\x12\x36\n\x07retries\x18\x05 \x01(\x0b\x32\x1c.flyteidl.core.RetryStrategyR\x07retries\x12+\n\x11\x64iscovery_version\x18\x06 \x01(\tR\x10\x64iscoveryVersion\x12\x38\n\x18\x64\x65precated_error_message\x18\x07 \x01(\tR\x16\x64\x65precatedErrorMessage\x12&\n\rinterruptible\x18\x08 \x01(\x08H\x00R\rinterruptible\x12-\n\x12\x63\x61\x63he_serializable\x18\t \x01(\x08R\x11\x63\x61\x63heSerializable\x12%\n\x0egenerates_deck\x18\n \x01(\x08R\rgeneratesDeck\x12\x39\n\x04tags\x18\x0b \x03(\x0b\x32%.flyteidl.core.TaskMetadata.TagsEntryR\x04tags\x12*\n\x11pod_template_name\x18\x0c \x01(\tR\x0fpodTemplateName\x12\x35\n\x17\x63\x61\x63he_ignore_input_vars\x18\r \x03(\tR\x14\x63\x61\x63heIgnoreInputVars\x12\x19\n\x08is_eager\x18\x0e \x01(\x08R\x07isEager\x1a\x37\n\tTagsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x15\n\x13interruptible_value\"\xd6\x05\n\x0cTaskTemplate\x12)\n\x02id\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.IdentifierR\x02id\x12\x12\n\x04type\x18\x02 \x01(\tR\x04type\x12\x37\n\x08metadata\x18\x03 \x01(\x0b\x32\x1b.flyteidl.core.TaskMetadataR\x08metadata\x12;\n\tinterface\x18\x04 \x01(\x0b\x32\x1d.flyteidl.core.TypedInterfaceR\tinterface\x12/\n\x06\x63ustom\x18\x05 \x01(\x0b\x32\x17.google.protobuf.StructR\x06\x63ustom\x12\x38\n\tcontainer\x18\x06 \x01(\x0b\x32\x18.flyteidl.core.ContainerH\x00R\tcontainer\x12\x30\n\x07k8s_pod\x18\x11 \x01(\x0b\x32\x15.flyteidl.core.K8sPodH\x00R\x06k8sPod\x12&\n\x03sql\x18\x12 \x01(\x0b\x32\x12.flyteidl.core.SqlH\x00R\x03sql\x12*\n\x11task_type_version\x18\x07 \x01(\x05R\x0ftaskTypeVersion\x12I\n\x10security_context\x18\x08 \x01(\x0b\x32\x1e.flyteidl.core.SecurityContextR\x0fsecurityContext\x12O\n\x12\x65xtended_resources\x18\t \x01(\x0b\x32 .flyteidl.core.ExtendedResourcesR\x11\x65xtendedResources\x12?\n\x06\x63onfig\x18\x10 \x03(\x0b\x32\'.flyteidl.core.TaskTemplate.ConfigEntryR\x06\x63onfig\x1a\x39\n\x0b\x43onfigEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x08\n\x06target\"6\n\rContainerPort\x12%\n\x0e\x63ontainer_port\x18\x01 \x01(\rR\rcontainerPort\"\xfc\x03\n\tContainer\x12\x14\n\x05image\x18\x01 \x01(\tR\x05image\x12\x18\n\x07\x63ommand\x18\x02 \x03(\tR\x07\x63ommand\x12\x12\n\x04\x61rgs\x18\x03 \x03(\tR\x04\x61rgs\x12\x36\n\tresources\x18\x04 \x01(\x0b\x32\x18.flyteidl.core.ResourcesR\tresources\x12-\n\x03\x65nv\x18\x05 \x03(\x0b\x32\x1b.flyteidl.core.KeyValuePairR\x03\x65nv\x12\x37\n\x06\x63onfig\x18\x06 \x03(\x0b\x32\x1b.flyteidl.core.KeyValuePairB\x02\x18\x01R\x06\x63onfig\x12\x32\n\x05ports\x18\x07 \x03(\x0b\x32\x1c.flyteidl.core.ContainerPortR\x05ports\x12\x41\n\x0b\x64\x61ta_config\x18\t \x01(\x0b\x32 .flyteidl.core.DataLoadingConfigR\ndataConfig\x12I\n\x0c\x61rchitecture\x18\n \x01(\x0e\x32%.flyteidl.core.Container.ArchitectureR\x0c\x61rchitecture\"I\n\x0c\x41rchitecture\x12\x0b\n\x07UNKNOWN\x10\x00\x12\t\n\x05\x41MD64\x10\x01\x12\t\n\x05\x41RM64\x10\x02\x12\n\n\x06\x41RM_V6\x10\x03\x12\n\n\x06\x41RM_V7\x10\x04\"\xb5\x02\n\nIOStrategy\x12K\n\rdownload_mode\x18\x01 \x01(\x0e\x32&.flyteidl.core.IOStrategy.DownloadModeR\x0c\x64ownloadMode\x12\x45\n\x0bupload_mode\x18\x02 \x01(\x0e\x32$.flyteidl.core.IOStrategy.UploadModeR\nuploadMode\"L\n\x0c\x44ownloadMode\x12\x12\n\x0e\x44OWNLOAD_EAGER\x10\x00\x12\x13\n\x0f\x44OWNLOAD_STREAM\x10\x01\x12\x13\n\x0f\x44O_NOT_DOWNLOAD\x10\x02\"E\n\nUploadMode\x12\x12\n\x0eUPLOAD_ON_EXIT\x10\x00\x12\x10\n\x0cUPLOAD_EAGER\x10\x01\x12\x11\n\rDO_NOT_UPLOAD\x10\x02\"\xa7\x02\n\x11\x44\x61taLoadingConfig\x12\x18\n\x07\x65nabled\x18\x01 \x01(\x08R\x07\x65nabled\x12\x1d\n\ninput_path\x18\x02 \x01(\tR\tinputPath\x12\x1f\n\x0boutput_path\x18\x03 \x01(\tR\noutputPath\x12I\n\x06\x66ormat\x18\x04 \x01(\x0e\x32\x31.flyteidl.core.DataLoadingConfig.LiteralMapFormatR\x06\x66ormat\x12:\n\x0bio_strategy\x18\x05 \x01(\x0b\x32\x19.flyteidl.core.IOStrategyR\nioStrategy\"1\n\x10LiteralMapFormat\x12\x08\n\x04JSON\x10\x00\x12\x08\n\x04YAML\x10\x01\x12\t\n\x05PROTO\x10\x02\"\xf3\x01\n\x06K8sPod\x12<\n\x08metadata\x18\x01 \x01(\x0b\x32 .flyteidl.core.K8sObjectMetadataR\x08metadata\x12\x32\n\x08pod_spec\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x07podSpec\x12\x41\n\x0b\x64\x61ta_config\x18\x03 \x01(\x0b\x32 .flyteidl.core.DataLoadingConfigR\ndataConfig\x12\x34\n\x16primary_container_name\x18\x04 \x01(\tR\x14primaryContainerName\"\xa9\x02\n\x11K8sObjectMetadata\x12\x44\n\x06labels\x18\x01 \x03(\x0b\x32,.flyteidl.core.K8sObjectMetadata.LabelsEntryR\x06labels\x12S\n\x0b\x61nnotations\x18\x02 \x03(\x0b\x32\x31.flyteidl.core.K8sObjectMetadata.AnnotationsEntryR\x0b\x61nnotations\x1a\x39\n\x0bLabelsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a>\n\x10\x41nnotationsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"\x92\x01\n\x03Sql\x12\x1c\n\tstatement\x18\x01 \x01(\tR\tstatement\x12\x34\n\x07\x64ialect\x18\x02 \x01(\x0e\x32\x1a.flyteidl.core.Sql.DialectR\x07\x64ialect\"7\n\x07\x44ialect\x12\r\n\tUNDEFINED\x10\x00\x12\x08\n\x04\x41NSI\x10\x01\x12\x08\n\x04HIVE\x10\x02\x12\t\n\x05OTHER\x10\x03\x42\xb0\x01\n\x11\x63om.flyteidl.coreB\nTasksProtoP\x01Z:github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core\xa2\x02\x03\x46\x43X\xaa\x02\rFlyteidl.Core\xca\x02\rFlyteidl\\Core\xe2\x02\x19\x46lyteidl\\Core\\GPBMetadata\xea\x02\x0e\x46lyteidl::Coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -77,15 +77,15 @@ _globals['_DATALOADINGCONFIG_LITERALMAPFORMAT']._serialized_start=3553 _globals['_DATALOADINGCONFIG_LITERALMAPFORMAT']._serialized_end=3602 _globals['_K8SPOD']._serialized_start=3605 - _globals['_K8SPOD']._serialized_end=3846 - _globals['_K8SOBJECTMETADATA']._serialized_start=3849 - _globals['_K8SOBJECTMETADATA']._serialized_end=4146 - _globals['_K8SOBJECTMETADATA_LABELSENTRY']._serialized_start=4025 - _globals['_K8SOBJECTMETADATA_LABELSENTRY']._serialized_end=4082 - _globals['_K8SOBJECTMETADATA_ANNOTATIONSENTRY']._serialized_start=4084 - _globals['_K8SOBJECTMETADATA_ANNOTATIONSENTRY']._serialized_end=4146 - _globals['_SQL']._serialized_start=4149 - _globals['_SQL']._serialized_end=4295 - _globals['_SQL_DIALECT']._serialized_start=4240 - _globals['_SQL_DIALECT']._serialized_end=4295 + _globals['_K8SPOD']._serialized_end=3848 + _globals['_K8SOBJECTMETADATA']._serialized_start=3851 + _globals['_K8SOBJECTMETADATA']._serialized_end=4148 + _globals['_K8SOBJECTMETADATA_LABELSENTRY']._serialized_start=4027 + _globals['_K8SOBJECTMETADATA_LABELSENTRY']._serialized_end=4084 + _globals['_K8SOBJECTMETADATA_ANNOTATIONSENTRY']._serialized_start=4086 + _globals['_K8SOBJECTMETADATA_ANNOTATIONSENTRY']._serialized_end=4148 + _globals['_SQL']._serialized_start=4151 + _globals['_SQL']._serialized_end=4297 + _globals['_SQL_DIALECT']._serialized_start=4242 + _globals['_SQL_DIALECT']._serialized_end=4297 # @@protoc_insertion_point(module_scope) diff --git a/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.pyi b/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.pyi index d7efcedbab..ac661ef4ca 100644 --- a/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.pyi +++ b/flyteidl/gen/pb_python/flyteidl/core/tasks_pb2.pyi @@ -232,16 +232,16 @@ class DataLoadingConfig(_message.Message): def __init__(self, enabled: bool = ..., input_path: _Optional[str] = ..., output_path: _Optional[str] = ..., format: _Optional[_Union[DataLoadingConfig.LiteralMapFormat, str]] = ..., io_strategy: _Optional[_Union[IOStrategy, _Mapping]] = ...) -> None: ... class K8sPod(_message.Message): - __slots__ = ["metadata", "pod_spec", "data_config", "primarycontainername"] + __slots__ = ["metadata", "pod_spec", "data_config", "primary_container_name"] METADATA_FIELD_NUMBER: _ClassVar[int] POD_SPEC_FIELD_NUMBER: _ClassVar[int] DATA_CONFIG_FIELD_NUMBER: _ClassVar[int] - PRIMARYCONTAINERNAME_FIELD_NUMBER: _ClassVar[int] + PRIMARY_CONTAINER_NAME_FIELD_NUMBER: _ClassVar[int] metadata: K8sObjectMetadata pod_spec: _struct_pb2.Struct data_config: DataLoadingConfig - primarycontainername: str - def __init__(self, metadata: _Optional[_Union[K8sObjectMetadata, _Mapping]] = ..., pod_spec: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., data_config: _Optional[_Union[DataLoadingConfig, _Mapping]] = ..., primarycontainername: _Optional[str] = ...) -> None: ... + primary_container_name: str + def __init__(self, metadata: _Optional[_Union[K8sObjectMetadata, _Mapping]] = ..., pod_spec: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., data_config: _Optional[_Union[DataLoadingConfig, _Mapping]] = ..., primary_container_name: _Optional[str] = ...) -> None: ... class K8sObjectMetadata(_message.Message): __slots__ = ["labels", "annotations"] diff --git a/flyteidl/gen/pb_rust/flyteidl.core.rs b/flyteidl/gen/pb_rust/flyteidl.core.rs index 5ba8b64819..041aee3937 100644 --- a/flyteidl/gen/pb_rust/flyteidl.core.rs +++ b/flyteidl/gen/pb_rust/flyteidl.core.rs @@ -1738,7 +1738,7 @@ pub struct K8sPod { pub data_config: ::core::option::Option, /// Defines the primary container name when pod template override is executed. #[prost(string, tag="4")] - pub primarycontainername: ::prost::alloc::string::String, + pub primary_container_name: ::prost::alloc::string::String, } /// Metadata for building a kubernetes object when a task is executed. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/flyteidl/protos/flyteidl/core/tasks.proto b/flyteidl/protos/flyteidl/core/tasks.proto index b0b7db5eaf..2e629f8b9b 100644 --- a/flyteidl/protos/flyteidl/core/tasks.proto +++ b/flyteidl/protos/flyteidl/core/tasks.proto @@ -323,7 +323,7 @@ message K8sPod { // Defines the primary container name when pod template override is executed. - string primarycontainername = 4; + string primary_container_name = 4; } // Metadata for building a kubernetes object when a task is executed. diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go index 13564ba0bc..4ab14ff64c 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -271,6 +271,7 @@ func BuildRawPod(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (*v Labels: make(map[string]string), } primaryContainerName := "" + switch target := taskTemplate.GetTarget().(type) { case *core.TaskTemplate_Container: // handles tasks defined by a single container @@ -286,9 +287,9 @@ func BuildRawPod(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (*v }, } if tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate() != nil { - if len(tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().Primarycontainername) > 0 { + if len(tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().PrimaryContainerName) > 0 { podSpec, objectMeta, err = ApplyPodTemplateOverride(podSpec, objectMeta, tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) - primaryContainerName = tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().Primarycontainername + primaryContainerName = tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().PrimaryContainerName } } @@ -343,7 +344,6 @@ func hasExternalLinkType(taskTemplate *core.TaskTemplate) bool { // configuration PodTemplate (if exists). func ApplyFlytePodConfiguration(ctx context.Context, tCtx pluginsCore.TaskExecutionContext, podSpec *v1.PodSpec, objectMeta *metav1.ObjectMeta, primaryContainerName string) (*v1.PodSpec, *metav1.ObjectMeta, error) { taskTemplate, err := tCtx.TaskReader().Read(ctx) - if err != nil { logger.Warnf(ctx, "failed to read task information when trying to construct Pod, err: %s", err.Error()) return nil, nil, err From b6e360c3bb804c4ef9f57ae60aacef5c46c22a94 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Wed, 15 Jan 2025 00:41:40 +0800 Subject: [PATCH 09/13] add unit test and fix error Signed-off-by: Nelson Chen --- flyteidl/clients/go/assets/admin.swagger.json | 2 +- .../gen/pb-es/flyteidl/core/workflow_pb.ts | 1 + .../gen/pb-go/flyteidl/core/workflow.pb.go | 1 + .../flyteidl/service/admin.swagger.json | 2 +- .../flyteidl/service/agent.swagger.json | 2 +- flyteidl/gen/pb_rust/flyteidl.core.rs | 1 + flyteidl/protos/flyteidl/core/workflow.proto | 2 +- .../pluginmachinery/flytek8s/pod_helper.go | 34 +++++++++------ .../flytek8s/pod_helper_test.go | 42 +++++++++++++++++++ 9 files changed, 71 insertions(+), 16 deletions(-) diff --git a/flyteidl/clients/go/assets/admin.swagger.json b/flyteidl/clients/go/assets/admin.swagger.json index 325e818aaa..53be7ae2fa 100644 --- a/flyteidl/clients/go/assets/admin.swagger.json +++ b/flyteidl/clients/go/assets/admin.swagger.json @@ -8337,7 +8337,7 @@ }, "pod_template": { "$ref": "#/definitions/coreK8sPod", - "title": "Override for the pod template used by task pods" + "title": "Override for the pod template used by task pods\n+optional" } }, "description": "Optional task node overrides that will be applied at task execution time." diff --git a/flyteidl/gen/pb-es/flyteidl/core/workflow_pb.ts b/flyteidl/gen/pb-es/flyteidl/core/workflow_pb.ts index e40f8ed12d..bbba15a0ba 100644 --- a/flyteidl/gen/pb-es/flyteidl/core/workflow_pb.ts +++ b/flyteidl/gen/pb-es/flyteidl/core/workflow_pb.ts @@ -1216,6 +1216,7 @@ export class TaskNodeOverrides extends Message { /** * Override for the pod template used by task pods + * +optional * * @generated from field: flyteidl.core.K8sPod pod_template = 4; */ diff --git a/flyteidl/gen/pb-go/flyteidl/core/workflow.pb.go b/flyteidl/gen/pb-go/flyteidl/core/workflow.pb.go index 61e6e52504..9a8b82080d 100644 --- a/flyteidl/gen/pb-go/flyteidl/core/workflow.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/core/workflow.pb.go @@ -1653,6 +1653,7 @@ type TaskNodeOverrides struct { // Override for the image used by task pods. ContainerImage string `protobuf:"bytes,3,opt,name=container_image,json=containerImage,proto3" json:"container_image,omitempty"` // Override for the pod template used by task pods + // +optional PodTemplate *K8SPod `protobuf:"bytes,4,opt,name=pod_template,json=podTemplate,proto3" json:"pod_template,omitempty"` } diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json index 325e818aaa..53be7ae2fa 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/admin.swagger.json @@ -8337,7 +8337,7 @@ }, "pod_template": { "$ref": "#/definitions/coreK8sPod", - "title": "Override for the pod template used by task pods" + "title": "Override for the pod template used by task pods\n+optional" } }, "description": "Optional task node overrides that will be applied at task execution time." diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json index 2207225195..94ec553459 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/agent.swagger.json @@ -1884,7 +1884,7 @@ }, "pod_template": { "$ref": "#/definitions/coreK8sPod", - "title": "Override for the pod template used by task pods" + "title": "Override for the pod template used by task pods\n+optional" } }, "description": "Optional task node overrides that will be applied at task execution time." diff --git a/flyteidl/gen/pb_rust/flyteidl.core.rs b/flyteidl/gen/pb_rust/flyteidl.core.rs index b127a30321..200499dd47 100644 --- a/flyteidl/gen/pb_rust/flyteidl.core.rs +++ b/flyteidl/gen/pb_rust/flyteidl.core.rs @@ -2782,6 +2782,7 @@ pub struct TaskNodeOverrides { #[prost(string, tag="3")] pub container_image: ::prost::alloc::string::String, /// Override for the pod template used by task pods + /// +optional #[prost(message, optional, tag="4")] pub pod_template: ::core::option::Option, } diff --git a/flyteidl/protos/flyteidl/core/workflow.proto b/flyteidl/protos/flyteidl/core/workflow.proto index 1f9dde45b4..dc9c155673 100644 --- a/flyteidl/protos/flyteidl/core/workflow.proto +++ b/flyteidl/protos/flyteidl/core/workflow.proto @@ -338,8 +338,8 @@ message TaskNodeOverrides { string container_image = 3; // Override for the pod template used by task pods + //+optional K8sPod pod_template = 4; - } // A structure that uniquely identifies a launch plan in the system. diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go index 4ab14ff64c..213a6f6753 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -286,7 +286,10 @@ func BuildRawPod(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (*v *c, }, } - if tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate() != nil { + + // handle pod template override + podTemplate := tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate() + if podTemplate != nil { if len(tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().PrimaryContainerName) > 0 { podSpec, objectMeta, err = ApplyPodTemplateOverride(podSpec, objectMeta, tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) primaryContainerName = tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().PrimaryContainerName @@ -318,6 +321,16 @@ func BuildRawPod(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (*v mergeMapInto(target.K8SPod.GetMetadata().GetAnnotations(), objectMeta.Annotations) mergeMapInto(target.K8SPod.GetMetadata().GetLabels(), objectMeta.Labels) } + + // handle pod template override + podTemplate := tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate() + if podTemplate != nil { + if len(tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().PrimaryContainerName) > 0 { + podSpec, objectMeta, err = ApplyPodTemplateOverride(podSpec, objectMeta, tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) + primaryContainerName = tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().PrimaryContainerName + } + } + default: return nil, nil, "", pluginserrors.Errorf(pluginserrors.BadTaskSpecification, "invalid TaskSpecification, unable to determine Pod configuration") @@ -453,23 +466,20 @@ func ApplyContainerImageOverride(podSpec *v1.PodSpec, containerImage string, pri } } -func ApplyPodTemplateOverride(podSpec *v1.PodSpec, objectMeta metav1.ObjectMeta, podtemplate *core.K8SPod) (*v1.PodSpec, metav1.ObjectMeta, error) { - if podtemplate.Metadata.Annotations != nil { - mergeMapInto(podtemplate.Metadata.Annotations, objectMeta.Annotations) +func ApplyPodTemplateOverride(podSpec *v1.PodSpec, objectMeta metav1.ObjectMeta, podTemplate *core.K8SPod) (*v1.PodSpec, metav1.ObjectMeta, error) { + if podTemplate.Metadata.Annotations != nil { + mergeMapInto(podTemplate.Metadata.Annotations, objectMeta.Annotations) } - if podtemplate.Metadata.Labels != nil { - mergeMapInto(podtemplate.Metadata.Labels, objectMeta.Labels) + if podTemplate.Metadata.Labels != nil { + mergeMapInto(podTemplate.Metadata.Labels, objectMeta.Labels) } var podspec_override *v1.PodSpec - err := utils.UnmarshalStructToObj(podtemplate.PodSpec, &podspec_override) + err := utils.UnmarshalStructToObj(podTemplate.PodSpec, &podspec_override) if err != nil { return nil, objectMeta, err } - if podspec_override.Containers == nil { - return podSpec, objectMeta, nil - } else { - return podspec_override, objectMeta, nil - } + + return podspec_override, objectMeta, nil } func addTolerationInPodSpec(podSpec *v1.PodSpec, toleration *v1.Toleration) *v1.PodSpec { diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go index 9b2f99f9fb..b9f7afe73b 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go @@ -25,6 +25,7 @@ import ( config1 "github.com/flyteorg/flyte/flytestdlib/config" "github.com/flyteorg/flyte/flytestdlib/config/viper" "github.com/flyteorg/flyte/flytestdlib/storage" + "github.com/flyteorg/flyte/flytestdlib/utils" ) func dummyTaskExecutionMetadata(resources *v1.ResourceRequirements, extendedResources *core.ExtendedResources, containerImage string, podtemplate *core.K8SPod) pluginsCore.TaskExecutionMetadata { @@ -1063,6 +1064,47 @@ func TestToK8sPodContainerImage(t *testing.T) { }) } +func TestPodTemplateOverride(t *testing.T) { + metadata := &core.K8SObjectMetadata{ + Labels: map[string]string{ + "l": "a", + }, + Annotations: map[string]string{ + "a": "b", + }, + } + + podSpec := v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "foo", + Image: "foo:latest", + Args: []string{"foo", "bar"}, + }, + }, + } + + podSpecStruct, err := utils.MarshalObjToStruct(podSpec) + assert.NoError(t, err) + + t.Run("Override pod template", func(t *testing.T) { + taskContext := dummyExecContext(dummyTaskTemplate(), &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1024m"), + }}, nil, "", &core.K8SPod{ + PrimaryContainerName: "foo", + PodSpec: podSpecStruct, + Metadata: metadata, + }) + p, m, _, err := ToK8sPodSpec(context.TODO(), taskContext) + assert.NoError(t, err) + assert.Equal(t, "a", m.Labels["l"]) + assert.Equal(t, "b", m.Annotations["a"]) + assert.Equal(t, "foo:latest", p.Containers[0].Image) + assert.Equal(t, "foo", p.Containers[0].Name) + }) +} + func TestToK8sPodExtendedResources(t *testing.T) { assert.NoError(t, config.SetK8sPluginConfig(&config.K8sPluginConfig{ GpuDeviceNodeLabel: "gpu-node-label", From de12b122ac14dcae1f81841e74e31f55d3564167 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 18 Jan 2025 11:10:57 +0800 Subject: [PATCH 10/13] fix type and implement metadataOverride in plugin_manager.go Signed-off-by: Nelson Chen --- .../pluginmachinery/flytek8s/pod_helper.go | 32 ++++++++--------- .../v1alpha1/mocks/ExecutableNode.go | 34 +++++++++++++++++++ .../nodes/task/k8s/plugin_manager.go | 4 +-- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go index 213a6f6753..e9ef6a1f5c 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -290,10 +290,12 @@ func BuildRawPod(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (*v // handle pod template override podTemplate := tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate() if podTemplate != nil { - if len(tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().PrimaryContainerName) > 0 { - podSpec, objectMeta, err = ApplyPodTemplateOverride(podSpec, objectMeta, tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) - primaryContainerName = tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().PrimaryContainerName + podSpec, err = ApplyPodTemplateOverride(tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) + if err != nil { + return nil, nil, "", err } + primaryContainerName = tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().GetPrimaryContainerName() + } case *core.TaskTemplate_K8SPod: @@ -325,10 +327,11 @@ func BuildRawPod(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (*v // handle pod template override podTemplate := tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate() if podTemplate != nil { - if len(tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().PrimaryContainerName) > 0 { - podSpec, objectMeta, err = ApplyPodTemplateOverride(podSpec, objectMeta, tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) - primaryContainerName = tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().PrimaryContainerName + podSpec, err = ApplyPodTemplateOverride(tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate()) + if err != nil { + return nil, nil, "", err } + primaryContainerName = tCtx.TaskExecutionMetadata().GetOverrides().GetPodTemplate().GetPrimaryContainerName() } default: @@ -466,20 +469,15 @@ func ApplyContainerImageOverride(podSpec *v1.PodSpec, containerImage string, pri } } -func ApplyPodTemplateOverride(podSpec *v1.PodSpec, objectMeta metav1.ObjectMeta, podTemplate *core.K8SPod) (*v1.PodSpec, metav1.ObjectMeta, error) { - if podTemplate.Metadata.Annotations != nil { - mergeMapInto(podTemplate.Metadata.Annotations, objectMeta.Annotations) - } - if podTemplate.Metadata.Labels != nil { - mergeMapInto(podTemplate.Metadata.Labels, objectMeta.Labels) - } - var podspec_override *v1.PodSpec - err := utils.UnmarshalStructToObj(podTemplate.PodSpec, &podspec_override) +func ApplyPodTemplateOverride(podTemplate *core.K8SPod) (*v1.PodSpec, error) { + // metadata override will be implemented in plugin_manager.go + var podSpecOverride *v1.PodSpec + err := utils.UnmarshalStructToObj(podTemplate.GetPodSpec(), &podSpecOverride) if err != nil { - return nil, objectMeta, err + return nil, err } - return podspec_override, objectMeta, nil + return podSpecOverride, nil } func addTolerationInPodSpec(podSpec *v1.PodSpec, toleration *v1.Toleration) *v1.PodSpec { diff --git a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/mocks/ExecutableNode.go b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/mocks/ExecutableNode.go index e6a03b7deb..2d63e294b2 100644 --- a/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/mocks/ExecutableNode.go +++ b/flytepropeller/pkg/apis/flyteworkflow/v1alpha1/mocks/ExecutableNode.go @@ -452,6 +452,40 @@ func (_m *ExecutableNode) GetOutputAlias() []v1alpha1.Alias { return r0 } +type ExecutableNode_GetPodTemplate struct { + *mock.Call +} + +func (_m ExecutableNode_GetPodTemplate) Return(_a0 *core.K8SPod) *ExecutableNode_GetPodTemplate { + return &ExecutableNode_GetPodTemplate{Call: _m.Call.Return(_a0)} +} + +func (_m *ExecutableNode) OnGetPodTemplate() *ExecutableNode_GetPodTemplate { + c_call := _m.On("GetPodTemplate") + return &ExecutableNode_GetPodTemplate{Call: c_call} +} + +func (_m *ExecutableNode) OnGetPodTemplateMatch(matchers ...interface{}) *ExecutableNode_GetPodTemplate { + c_call := _m.On("GetPodTemplate", matchers...) + return &ExecutableNode_GetPodTemplate{Call: c_call} +} + +// GetPodTemplate provides a mock function with given fields: +func (_m *ExecutableNode) GetPodTemplate() *core.K8SPod { + ret := _m.Called() + + var r0 *core.K8SPod + if rf, ok := ret.Get(0).(func() *core.K8SPod); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*core.K8SPod) + } + } + + return r0 +} + type ExecutableNode_GetResources struct { *mock.Call } diff --git a/flytepropeller/pkg/controller/nodes/task/k8s/plugin_manager.go b/flytepropeller/pkg/controller/nodes/task/k8s/plugin_manager.go index 6d558f818e..fd1a87baac 100644 --- a/flytepropeller/pkg/controller/nodes/task/k8s/plugin_manager.go +++ b/flytepropeller/pkg/controller/nodes/task/k8s/plugin_manager.go @@ -112,8 +112,8 @@ type PluginManager struct { func (e *PluginManager) addObjectMetadata(taskCtx pluginsCore.TaskExecutionMetadata, o client.Object, cfg *config.K8sPluginConfig) { o.SetNamespace(taskCtx.GetNamespace()) - o.SetAnnotations(pluginsUtils.UnionMaps(cfg.DefaultAnnotations, o.GetAnnotations(), pluginsUtils.CopyMap(taskCtx.GetAnnotations()))) - o.SetLabels(pluginsUtils.UnionMaps(cfg.DefaultLabels, o.GetLabels(), pluginsUtils.CopyMap(taskCtx.GetLabels()))) + o.SetAnnotations(pluginsUtils.UnionMaps(cfg.DefaultAnnotations, o.GetAnnotations(), pluginsUtils.CopyMap(taskCtx.GetAnnotations()), taskCtx.GetOverrides().GetPodTemplate().GetMetadata().GetAnnotations())) + o.SetLabels(pluginsUtils.UnionMaps(cfg.DefaultLabels, o.GetLabels(), pluginsUtils.CopyMap(taskCtx.GetLabels()), taskCtx.GetOverrides().GetPodTemplate().GetMetadata().GetLabels())) o.SetName(taskCtx.GetTaskExecutionID().GetGeneratedName()) if !e.plugin.GetProperties().DisableInjectOwnerReferences { From 87d11be7d2d0fa2d3448a4edb077fec5382a7ee9 Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 18 Jan 2025 11:14:25 +0800 Subject: [PATCH 11/13] remove metadatest in podtemplate Signed-off-by: Nelson Chen --- .../pluginmachinery/flytek8s/pod_helper_test.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go index b9f7afe73b..c12a5c0426 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go @@ -1065,15 +1065,6 @@ func TestToK8sPodContainerImage(t *testing.T) { } func TestPodTemplateOverride(t *testing.T) { - metadata := &core.K8SObjectMetadata{ - Labels: map[string]string{ - "l": "a", - }, - Annotations: map[string]string{ - "a": "b", - }, - } - podSpec := v1.PodSpec{ Containers: []v1.Container{ { @@ -1094,12 +1085,9 @@ func TestPodTemplateOverride(t *testing.T) { }}, nil, "", &core.K8SPod{ PrimaryContainerName: "foo", PodSpec: podSpecStruct, - Metadata: metadata, }) - p, m, _, err := ToK8sPodSpec(context.TODO(), taskContext) + p, _, _, err := ToK8sPodSpec(context.TODO(), taskContext) assert.NoError(t, err) - assert.Equal(t, "a", m.Labels["l"]) - assert.Equal(t, "b", m.Annotations["a"]) assert.Equal(t, "foo:latest", p.Containers[0].Image) assert.Equal(t, "foo", p.Containers[0].Name) }) From fb9eebe54b8d1d0f96f05b4e3bab9411cb996b0e Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 18 Jan 2025 11:17:49 +0800 Subject: [PATCH 12/13] typo Signed-off-by: Nelson Chen --- .../go/tasks/pluginmachinery/flytek8s/pod_helper_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go index c12a5c0426..12115b2f38 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go @@ -28,7 +28,7 @@ import ( "github.com/flyteorg/flyte/flytestdlib/utils" ) -func dummyTaskExecutionMetadata(resources *v1.ResourceRequirements, extendedResources *core.ExtendedResources, containerImage string, podtemplate *core.K8SPod) pluginsCore.TaskExecutionMetadata { +func dummyTaskExecutionMetadata(resources *v1.ResourceRequirements, extendedResources *core.ExtendedResources, containerImage string, podTemplate *core.K8SPod) pluginsCore.TaskExecutionMetadata { taskExecutionMetadata := &pluginsCoreMock.TaskExecutionMetadata{} taskExecutionMetadata.On("GetNamespace").Return("test-namespace") taskExecutionMetadata.On("GetAnnotations").Return(map[string]string{"annotation-1": "val1"}) @@ -55,7 +55,7 @@ func dummyTaskExecutionMetadata(resources *v1.ResourceRequirements, extendedReso to.On("GetResources").Return(resources) to.On("GetExtendedResources").Return(extendedResources) to.On("GetContainerImage").Return(containerImage) - to.On("GetPodTemplate").Return(podtemplate) + to.On("GetPodTemplate").Return(podTemplate) taskExecutionMetadata.On("GetOverrides").Return(to) taskExecutionMetadata.On("IsInterruptible").Return(true) taskExecutionMetadata.OnGetPlatformResources().Return(&v1.ResourceRequirements{}) @@ -84,7 +84,7 @@ func dummyInputReader() io.InputReader { return inputReader } -func dummyExecContext(taskTemplate *core.TaskTemplate, r *v1.ResourceRequirements, rm *core.ExtendedResources, containerImage string, podtemplate *core.K8SPod) pluginsCore.TaskExecutionContext { +func dummyExecContext(taskTemplate *core.TaskTemplate, r *v1.ResourceRequirements, rm *core.ExtendedResources, containerImage string, podTemplate *core.K8SPod) pluginsCore.TaskExecutionContext { ow := &pluginsIOMock.OutputWriter{} ow.OnGetOutputPrefixPath().Return("") ow.OnGetRawOutputPrefix().Return("") @@ -92,7 +92,7 @@ func dummyExecContext(taskTemplate *core.TaskTemplate, r *v1.ResourceRequirement ow.OnGetPreviousCheckpointsPrefix().Return("/prev") tCtx := &pluginsCoreMock.TaskExecutionContext{} - tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(r, rm, containerImage, podtemplate)) + tCtx.OnTaskExecutionMetadata().Return(dummyTaskExecutionMetadata(r, rm, containerImage, podTemplate)) tCtx.OnInputReader().Return(dummyInputReader()) tCtx.OnOutputWriter().Return(ow) From 3276d4d23c50ea961526ade31307f9f12450c5da Mon Sep 17 00:00:00 2001 From: Nelson Chen Date: Sat, 18 Jan 2025 12:17:03 +0800 Subject: [PATCH 13/13] add test for override metadata in plugin_manager Signed-off-by: Nelson Chen --- .../nodes/task/k8s/plugin_manager_test.go | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/flytepropeller/pkg/controller/nodes/task/k8s/plugin_manager_test.go b/flytepropeller/pkg/controller/nodes/task/k8s/plugin_manager_test.go index a02c32dca1..9151a4311b 100644 --- a/flytepropeller/pkg/controller/nodes/task/k8s/plugin_manager_test.go +++ b/flytepropeller/pkg/controller/nodes/task/k8s/plugin_manager_test.go @@ -207,10 +207,22 @@ func getMockTaskExecutionMetadata() pluginsCore.TaskExecutionMetadata { taskExecutionMetadata.On("GetOwnerReference").Return(metav1.OwnerReference{Name: "x"}) taskExecutionMetadata.On("GetSecurityContext").Return(core.SecurityContext{RunAs: &core.Identity{}}) + metadata := &core.K8SObjectMetadata{ + Labels: map[string]string{}, + Annotations: map[string]string{}, + } + to := &pluginsCoreMock.TaskOverrides{} + to.On("GetPodTemplate").Return(&core.K8SPod{ + Metadata: metadata, + }, + ) + taskExecutionMetadata.On("GetOverrides").Return(to) + id := &pluginsCoreMock.TaskExecutionID{} id.On("GetGeneratedName").Return("test") id.On("GetID").Return(core.TaskExecutionIdentifier{}) taskExecutionMetadata.On("GetTaskExecutionID").Return(id) + return taskExecutionMetadata } @@ -219,13 +231,26 @@ func getMockTaskExecutionMetadataCustom( ns string, annotations map[string]string, labels map[string]string, - ownerRef metav1.OwnerReference) pluginsCore.TaskExecutionMetadata { + ownerRef metav1.OwnerReference, + annotationsOverride map[string]string, + labelsOverride map[string]string) pluginsCore.TaskExecutionMetadata { taskExecutionMetadata := &pluginsCoreMock.TaskExecutionMetadata{} taskExecutionMetadata.On("GetNamespace").Return(ns) taskExecutionMetadata.On("GetAnnotations").Return(annotations) taskExecutionMetadata.On("GetLabels").Return(labels) taskExecutionMetadata.On("GetOwnerReference").Return(ownerRef) + metadata := &core.K8SObjectMetadata{ + Labels: labelsOverride, + Annotations: annotationsOverride, + } + to := &pluginsCoreMock.TaskOverrides{} + to.On("GetPodTemplate").Return(&core.K8SPod{ + Metadata: metadata, + }, + ) + taskExecutionMetadata.On("GetOverrides").Return(to) + id := &pluginsCoreMock.TaskExecutionID{} id.On("GetGeneratedName").Return(tid) id.On("GetID").Return(core.TaskExecutionIdentifier{}) @@ -908,7 +933,9 @@ func TestPluginManager_AddObjectMetadata(t *testing.T) { or := metav1.OwnerReference{} l := map[string]string{"l1": "lv1"} a := map[string]string{"aKey": "aVal"} - tm := getMockTaskExecutionMetadataCustom(genName, ns, a, l, or) + lo := map[string]string{} + ao := map[string]string{} + tm := getMockTaskExecutionMetadataCustom(genName, ns, a, l, or, ao, lo) cfg := config.GetK8sPluginConfig() @@ -1008,6 +1035,25 @@ func TestPluginManager_AddObjectMetadata(t *testing.T) { assert.Contains(t, o.GetFinalizers(), finalizer) }) + t.Run("with override", func(t *testing.T) { + lo = map[string]string{"l1": "lv2"} + ao = map[string]string{"aKey": "bVal"} + tm = getMockTaskExecutionMetadataCustom(genName, ns, a, l, or, ao, lo) + + cfg = config.GetK8sPluginConfig() + + o := &v1.Pod{} + p := pluginsk8sMock.Plugin{} + p.OnGetProperties().Return(k8s.PluginProperties{}) + pluginManager := PluginManager{plugin: &p} + pluginManager.addObjectMetadata(tm, o, cfg) + assert.Equal(t, map[string]string{ + "cluster-autoscaler.kubernetes.io/safe-to-evict": "false", + "aKey": "bVal", + }, o.GetAnnotations()) + assert.Equal(t, lo, o.GetLabels()) + }) + } func TestResourceManagerConstruction(t *testing.T) {