migrate crd training job struct from cloud

.travis.yml

@@ -11,6 +11,9 @@ matrix:
         - rm -f .copyright.hook && wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/.copyright.hook
         - bash -x .tools/check_style.sh
         - ln -s $GOPATH/src/github.com/PaddlePaddle $GOPATH/src/github.com/paddlepaddle
+        - glide install
+        - vendor/k8s.io/code-generator/generate-groups.sh "deepcopy,client,informer,lister" github.com/PaddlePaddle/edl/pkg/client github.com/PaddlePaddle/edl/pkg/apis paddlepaddle:v1 --go-header-file scripts/custom-boilerplate.go.txt
+        - grep "github.com/paddlepaddle/edl" -nR pkg/client | awk -F ':' '{print $1}' | xargs sed -i 's|github.com/paddlepaddle/edl|github.com/PaddlePaddle/edl|g'
         - cd $GOPATH/src/github.com/paddlepaddle/edl
         - glide install --strip-vendor
         - go test $(glide novendor)

glide.yaml

@@ -1,4 +1,4 @@
-package: github.com/paddlepaddle/edl
+package: github.com/PaddlePaddle/edl
 import:
 - package: k8s.io/client-go
   version: kubernetes-1.8.0
@@ -9,3 +9,5 @@ import:
 - package: github.com/inconshreveable/log15
   version: v2.13
 - package: github.com/wangkuiyi/candy
+- package: k8s.io/code-generator
+  version: kubernetes-1.8.6

pkg/apis/paddlepaddle/register.go

@@ -0,0 +1,19 @@
+/*
+Copyright (c) 2016 PaddlePaddle Authors All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package paddlepaddle
+
+const (
+	// GroupName is the name of group.
+	GroupName = "paddlepaddle.org"
+)

pkg/apis/paddlepaddle/v1/doc.go

@@ -0,0 +1,21 @@
+/*
+Copyright (c) 2016 PaddlePaddle Authors All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:deepcopy-gen=package
+
+// Package v1 is the v1 version of the API.
+// +groupName=paddlepaddle.org
+package v1

pkg/apis/paddlepaddle/v1/register.go

@@ -0,0 +1,32 @@
+package v1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var (
+	// SchemeBuilder will call register
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	// AddToScheme will apply all the stored functions to the scheme
+	AddToScheme = SchemeBuilder.AddToScheme
+)
+
+// SchemeGroupVersion is the group version used to register these objects.
+var SchemeGroupVersion = schema.GroupVersion{Group: CRDGroup, Version: CRDVersion}
+
+// Resource takes an unqualified resource and returns a Group-qualified GroupResource.
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+// addKnownTypes adds the set of types defined in this package to the supplied scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&TrainingJob{},
+		&TrainingJobList{},
+	)
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+	return nil
+}

pkg/apis/paddlepaddle/v1/types.go

@@ -0,0 +1,171 @@
+package v1
+
+import (
+	"fmt"
+
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	v1beta1 "k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	// CRDKind is the kind of K8s CRD.
+	CRDKind = "TrainingJob"
+	// CRDKindPlural is the plural of CRDKind.
+	CRDKindPlural = "trainingjobs"
+	// CRDShortName is the short name of CRD.
+	CRDShortName = "tj"
+	// CRDGroup is the name of group.
+	CRDGroup = "paddlepaddle.org"
+	// CRDVersion is the version of CRD.
+	CRDVersion = "v1"
+)
+
+// CRDName returns name of crd
+func CRDName() string {
+	return fmt.Sprintf("%s.%s", CRDKindPlural, CRDGroup)
+}
+
+// +genclient
+// +genclient:noStatus
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +resource:path=trainingjob
+
+// TrainingJob is a specification for a TrainingJob resource
+type TrainingJob struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Spec              TrainingJobSpec   `json:"spec"`
+	Status            TrainingJobStatus `json:"status"`
+}
+
+// TrainingJobSpec is the spec for a TrainingJob resource
+type TrainingJobSpec struct {
+	// General job attributes.
+	Image string `json:"image,omitempty"`
+	// If you want to use the hostnetwork instead of container network
+	// portmanager is necessary. About portmanager, please refer to
+	// https://github.com/PaddlePaddle/cloud/blob/develop/doc/hostnetwork/hostnetwork.md
+	HostNetwork       bool                 `json:"host_network,omitempty"`
+	Port              int                  `json:"port,omitempty"`
+	PortsNum          int                  `json:"ports_num,omitempty"`
+	PortsNumForSparse int                  `json:"ports_num_for_sparse,omitempty"`
+	FaultTolerant     bool                 `json:"fault_tolerant,omitempty"`
+	Passes            int                  `json:"passes,omitempty"`
+	Volumes           []corev1.Volume      `json:"volumes"`
+	VolumeMounts      []corev1.VolumeMount `json:"VolumeMounts"`
+	//TrainingJob components.
+	Master  MasterSpec  `json:"master"`
+	Pserver PserverSpec `json:"pserver"`
+	Trainer TrainerSpec `json:"trainer"`
+}
+
+// MasterSpec is the spec for a master in the paddle job
+type MasterSpec struct {
+	EtcdEndpoint string                      `json:"etcd-endpoint"`
+	Resources    corev1.ResourceRequirements `json:"resources"`
+	ReplicaSpec  *v1beta1.ReplicaSet         `json:"replicaSpec"`
+}
+
+// PserverSpec is the spec for pservers in the paddle job
+type PserverSpec struct {
+	MinInstance int                         `json:"min-instance"`
+	MaxInstance int                         `json:"max-instance"`
+	Resources   corev1.ResourceRequirements `json:"resources"`
+	ReplicaSpec *v1beta1.ReplicaSet         `json:"replicaSpec"`
+}
+
+// TrainerSpec is the spec for trainers in the paddle job
+type TrainerSpec struct {
+	EtcdEndpoint string                      `json:"etcd-endpoint"`
+	Entrypoint   string                      `json:"entrypoint"`
+	Workspace    string                      `json:"workspace"`
+	MinInstance  int                         `json:"min-instance"`
+	MaxInstance  int                         `json:"max-instance"`
+	Resources    corev1.ResourceRequirements `json:"resources"`
+	ReplicaSpec  *batchv1.Job                `json:"replicaSpec"`
+}
+
+// TrainingJobPhase is the phase of TrainingJob
+type TrainingJobPhase string
+
+const (
+	// TrainingJobPhaseNone is empty TrainingJobPhase.
+	TrainingJobPhaseNone TrainingJobPhase = ""
+	// TrainingJobPhaseCreating is creating TrainingJobPhase.
+	TrainingJobPhaseCreating = "creating"
+	// TrainingJobPhaseRunning is running TrainingJobPhase.
+	TrainingJobPhaseRunning = "running"
+	// TrainingJobPhaseSucceeded is succeeded TrainingJobPhase.
+	TrainingJobPhaseSucceeded = "succeeded"
+	// TrainingJobPhaseFailed is failed TrainingJobPhase.
+	TrainingJobPhaseFailed = "failed"
+)
+
+// TrainerJobScaleStatus is status of trainer jobs.
+type TrainerJobScaleStatus struct {
+}
+
+// TrainingResourceType the type of TrainingJob resource, include MASTER PSERVER and TRAINER
+type TrainingResourceType string
+
+const (
+	// MASTER is the master name of TrainingResourceType.
+	MASTER TrainingResourceType = "MASTER"
+	// PSERVER is the pserver name of TrainingResourceType.
+	PSERVER TrainingResourceType = "PSERVER"
+	// TRAINER is the trainer name of TrainingResourceType.
+	TRAINER TrainingResourceType = "TRAINER"
+)
+
+// ResourceState is the state of a type of resource
+type ResourceState string
+
+const (
+	// ResourceStateNone is the initial state of training job
+	ResourceStateNone ResourceState = ""
+	// ResourceStateStarting is the starting state of ResourceState.
+	ResourceStateStarting = "starting"
+	// ResourceStateRunning is the  running state of ResourceState.
+	ResourceStateRunning = "running"
+	// ResourceStateFailed is the failed state of ResourceState.
+	ResourceStateFailed = "failed"
+	// ResourceStateSucceeded is the succeeded state of ResourceState
+	ResourceStateSucceeded = "succeeded"
+)
+
+// TrainingResourceStatus is the status of every resource
+type TrainingResourceStatus struct {
+	// TrainingResourceType the type of TrainingJob resource, include MASTER PSERVER and TRAINER
+	TrainingResourceType `json:"training_resource_type"`
+	// State is the state of a type of resource
+	State ResourceState `json:"state"`
+	// ResourceStates is the number of resource in different state
+	ResourceStates map[ResourceState]int `json:"resource_states"`
+}
+
+// TrainingJobStatus is the status for a TrainingJob resource.
+type TrainingJobStatus struct {
+	// Phase is phase of TrainingJob
+	Phase TrainingJobPhase `json:"phase"`
+	// Reason is the reason of job phase failed
+	Reason string `json:"reason"`
+	// ScaleStatus is autoscale status of trainer jobs
+	// TODO(ZhengQi): this will used in autoscale mode in future.
+	ScaleStatus TrainerJobScaleStatus `json:"scale_status"`
+	// ReplicaStatuses is detail status of resources
+	// TODO(ZhengQi): should we only considered trainer job now?
+	ReplicaStatuses []*TrainingResourceStatus `json:"replica_statuses"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +resource:path=trainingjobs
+
+// TrainingJobList is a list of TrainingJob resources
+type TrainingJobList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+	// Items means the list of paddle job/TrainingJob
+	Items []TrainingJob `json:"items"`
+}

scripts/custom-boilerplate.go.txt

@@ -0,0 +1,15 @@
+/* 
+Copyright (c) 2016 PaddlePaddle Authors All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and 
+limitations under the License.
+*/

scripts/update-codegen.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This shell is used to auto generate some useful tools for k8s, such as lister,
+# informer, deepcopy, defaulter and so on.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
+echo ${SCRIPT_ROOT}
+CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 vendor/k8s.io/code-generator 2>/dev/null || echo code-generator)}
+echo ${CODEGEN_PKG}
+
+# generate the code with:
+# --output-base    because this script should also be able to run inside the vendor dir of
+#                  k8s.io/kubernetes. The output-base is needed for the generators to output into the vendor dir
+#                  instead of the $GOPATH directly. For normal projects this can be dropped.
+${CODEGEN_PKG}/generate-groups.sh "deepcopy,client,informer,lister" \
+ github.com/PaddlePaddle/edl/pkg/client github.com/PaddlePaddle/edl/pkg/apis \
+  paddlepaddle:v1 \
+  --go-header-file ${SCRIPT_ROOT}/scripts/custom-boilerplate.go.txt
+
+grep "github.com/paddlepaddle/edl" -nR pkg/client | awk -F ':' '{print $1}' | \
+ xargs sed -i "" 's|github.com/paddlepaddle/edl|github.com/PaddlePaddle/edl|g'
+
+## format codes
+gofmt -w pkg/client