Merge branch 'kdkasrav/k8s-improvements' into 'develop'

k8s kustomize refactoring and improvements

See merge request intelai/models!218

examples/common/KubernetesMlOps.md

@@ -0,0 +1,28 @@
+# Kubernetes Mlops
+
+## Using NFS as storage
+
+Configuring the examples to use NFS as storage requires specifying these values in the mlops.env within the tree show below:
+
+```
+examples
+└── common
+    └── k8s
+        └── mlops
+            ├── base
+            │   └──  mlops.env
+            ├── multi-node
+            └── single-node
+```
+
+The NFS related values within mlops.env are shown below:
+
+```
+NFS_PATH=/exported_users
+NFS_MOUNT_PATH=/home
+NFS_SERVER=0.0.0.0
+```
+
+They should reflect values specific to your NFS implementation. NFS_PATH and NFS_SERVER are typically found in /etc/mtab 
+and are NFS server values. NFS_MOUNT_PATH is a nfs client option indicating where the exported file system is mounted at.
+

examples/common/k8s/mlops/base/mlops.env

@@ -0,0 +1,5 @@
+DATASET_DIR=/dataset
+NFS_PATH=/exported_users
+NFS_MOUNT_PATH=/home
+NFS_SERVER=0.0.0.0
+OUTPUT_DIR=/workspace/output

examples/common/k8s/mlops/multi-node/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../base
+resources:
+- mpi-job.yaml
+configurations:
+- mlops.yaml

examples/image_recognition/tensorflow/resnet50v1_5/training/fp32/k8s/mlops/multi-node/mlops.yaml → examples/common/k8s/mlops/multi-node/mlops.yaml

@@ -1,4 +1,6 @@
 varReference:
+- kind: MPIJob
+  path: metadata/name
 - kind: MPIJob
   path: spec/mpiReplicaSpecs/Launcher/template/spec/containers/image
 - kind: MPIJob

examples/language_modeling/tensorflow/bert_large/training/fp32/k8s/mlops/multi-node/mpi-job.yaml → examples/common/k8s/mlops/multi-node/mpi-job.yaml

@@ -1,7 +1,7 @@
 apiVersion: kubeflow.org/v1alpha2
 kind: MPIJob
 metadata:
-  name: bert-large-fp32-training
+  name: $(MODEL_NAME)
 spec:
   slotsPerWorker: 1
   cleanPodPolicy: Running
@@ -13,17 +13,8 @@ spec:
           serviceAccountName: model-service
           containers:
           - name: mpi-launcher
-            image: $(REGISTRY)/model-zoo:2.1.0-language-modeling-bert-large-fp32-training
             imagePullPolicy: Always
             workingDir: /
-            command:
-            - $(MODEL_DIR)/examples/fp32_training_multi_node.sh
-            volumeMounts:
-            - name: datasets
-              mountPath: $(DATASET_DIR)
-              readOnly: true
-            - name: users
-              mountPath: /Users
           volumes:
           - name: datasets
             persistentVolumeClaim:
@@ -38,15 +29,8 @@ spec:
           serviceAccountName: model-service
           containers:
           - name: mpi-worker
-            image: $(REGISTRY)/model-zoo:2.1.0-language-modeling-bert-large-fp32-training
             imagePullPolicy: Always
             workingDir: /
-            volumeMounts:
-            - name: datasets
-              mountPath: $(DATASET_DIR)
-              readOnly: true
-            - name: users
-              mountPath: /Users
           volumes:
           - name: datasets
             persistentVolumeClaim:

examples/common/k8s/mlops/single-node/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../base
+resources:
+- pod.yaml
+configurations:
+- mlops.yaml

examples/language_modeling/tensorflow/bert_large/training/fp32/k8s/mlops/single-node/mlops.yaml → examples/common/k8s/mlops/single-node/mlops.yaml

@@ -1,6 +1,8 @@
 varReference:
 - kind: PersistentVolume
   path: spec/hostPath/path
+- kind: Pod
+  path: metadata/name
 - kind: Pod
   path: spec/containers/image
 - kind: Pod

examples/common/k8s/mlops/single-node/pod.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: $(MODEL_NAME)
+spec:
+  serviceAccountName: model-service
+  containers:
+  - name: single-node
+    imagePullPolicy: Always
+  volumes:
+  - name: users
+    persistentVolumeClaim:
+      claimName: users-pvc
+  - name: datasets
+    persistentVolumeClaim:
+      claimName: datasets-pvc

examples/image_recognition/tensorflow/resnet50v1_5/training/fp32/k8s/mlops/multi-node/kustomization.yaml

@@ -1,13 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
-- ../base
-resources:
-- mpi-job.yaml
-images:
-- name: model-zoo
-  newName: model-zoo
-  newTag: 2.1.0-image-recognition-resnet50v1-5-fp32-training
+- ../../../common/k8s/mlops/multi-node
 configMapGenerator:
 - name: mlops-env
   env: mlops.env
@@ -17,19 +11,31 @@ generatorOptions:
 patchesStrategicMerge:
 - mpi-job-patch.yaml
 vars:
+- name: MODEL_NAME
+  fieldref:
+    fieldPath: data.MODEL_NAME
+  objref:
+    apiVersion: v1
+    kind: ConfigMap
+    name: mlops-env
 - name: MODEL_DIR
   fieldref:
     fieldPath: data.MODEL_DIR
   objref:
     apiVersion: v1
     kind: ConfigMap
     name: mlops-env
+- name: NFS_MOUNT_PATH
+  fieldref:
+    fieldPath: data.NFS_MOUNT_PATH
+  objref:
+    apiVersion: v1
+    kind: ConfigMap
+    name: mlops-env
 - name: REGISTRY
   fieldref:
     fieldPath: data.REGISTRY
   objref:
     apiVersion: v1
     kind: ConfigMap
     name: mlops-env
-configurations:
-- mlops.yaml

examples/image_recognition/tensorflow/resnet50v1_5/training/fp32/k8s/mlops/multi-node/mlops.env

@@ -1,4 +1,6 @@
 DATASET_DIR=/tf_dataset
+MODEL_NAME=resnet50v1_5_fp32_training
 MODEL_DIR=/workspace/resnet50v1_5_fp32_training
+NFS_MOUNT_PATH=/Users
 OUTPUT_DIR=/tmp/output
 REGISTRY=amr-registry.caas.intel.com/aipg-tf

examples/image_recognition/tensorflow/resnet50v1_5/training/fp32/k8s/mlops/multi-node/mpi-job-patch.yaml

@@ -1,7 +1,7 @@
 apiVersion: kubeflow.org/v1alpha2
 kind: MPIJob
 metadata:
-  name: resnet50v1-5-fp32-training
+  name: $(MODEL_NAME)
 spec:
   mpiReplicaSpecs:
     Launcher:
@@ -10,23 +10,21 @@ spec:
           containers:
           - name: mpi-launcher
             image: $(REGISTRY)/model-zoo:2.1.0-image-recognition-resnet50v1-5-fp32-training
-            imagePullPolicy: Always
             securityContext:
               runAsUser: 0
               runAsGroup: 0
               fsGroup: 0
             envFrom:
             - configMapRef:
                 name: mlops-env
-            workingDir: /
             command:
             - $(MODEL_DIR)/examples/fp32_training_multi_node.sh
             volumeMounts:
             - name: datasets
               mountPath: $(DATASET_DIR)
               readOnly: true
             - name: users
-              mountPath: /Users
+              mountPath: $(NFS_MOUNT_PATH)
     Worker:
       template:
         spec:
@@ -46,4 +44,4 @@ spec:
               mountPath: $(DATASET_DIR)
               readOnly: true
             - name: users
-              mountPath: /Users
+              mountPath: $(NFS_MOUNT_PATH)

examples/image_recognition/tensorflow/resnet50v1_5/training/fp32/k8s/mlops/single-node/kustomization.yaml

@@ -1,13 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
-- ../base
-resources:
-- pod.yaml
-images:
-- name: model-zoo
-  newName: model-zoo
-  newTag: 2.1.0-image-recognition-resnet50v1-5-fp32-training
+- ../../../common/k8s/mlops/single-node
 configMapGenerator:
 - name: mlops-env
   env: mlops.env
@@ -17,19 +11,31 @@ generatorOptions:
 patchesStrategicMerge:
 - pod-patch.yaml
 vars:
+- name: MODEL_NAME
+  fieldref:
+    fieldPath: data.MODEL_NAME
+  objref:
+    apiVersion: v1
+    kind: ConfigMap
+    name: mlops-env
 - name: MODEL_DIR
   fieldref:
     fieldPath: data.MODEL_DIR
   objref:
     apiVersion: v1
     kind: ConfigMap
     name: mlops-env
+- name: NFS_MOUNT_PATH
+  fieldref:
+    fieldPath: data.NFS_MOUNT_PATH
+  objref:
+    apiVersion: v1
+    kind: ConfigMap
+    name: mlops-env
 - name: REGISTRY
   fieldref:
     fieldPath: data.REGISTRY
   objref:
     apiVersion: v1
     kind: ConfigMap
     name: mlops-env
-configurations:
-- mlops.yaml

examples/image_recognition/tensorflow/resnet50v1_5/training/fp32/k8s/mlops/single-node/mlops.env

@@ -1,4 +1,6 @@
 DATASET_DIR=/tf_dataset
+MODEL_NAME=resnet50v1_5_fp32_training
 MODEL_DIR=/workspace/resnet50v1_5_fp32_training
+/NFS_MOUNT_PATH=/Users
 OUTPUT_DIR=/tmp/output
 REGISTRY=amr-registry.caas.intel.com/aipg-tf

examples/image_recognition/tensorflow/resnet50v1_5/training/fp32/k8s/mlops/single-node/pod-patch.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: resnet50v1-5-fp32-training
+  name: $(MODEL_NAME)
 spec:
   securityContext:
     runAsUser: 0
@@ -16,4 +16,10 @@ spec:
         name: mlops-env
     workingDir: /
     command:
-    - $(MODEL_DIR)/examples/fp32_training_multi_node.sh
+    - $(MODEL_DIR)/examples/fp32_training_demo.sh
+    volumeMounts:
+    - name: datasets
+      mountPath: $(DATASET_DIR)
+      readOnly: true
+    - name: users
+      mountPath: $(NFS_MOUNT_PATH)