From 63eb0037d52990cf4fb01fcce3accdb34acc096d Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Fri, 5 Jun 2020 00:44:08 -0400 Subject: [PATCH] add document to the new io chaos template mechanism (#545) Signed-off-by: yeya24 Co-authored-by: CWen --- docs/io_chaos.md | 43 +++++--- docs/sidecar_configmap.md | 61 +++++++---- docs/template_config.md | 103 ++++++++++++++++++ examples/chaosfs-configmap/pd-configmap.yaml | 2 +- .../chaosfs-configmap/tiflash-configmap.yaml | 2 +- .../chaosfs-configmap/tikv-configmap.yaml | 2 +- examples/etcd/etcd-configmap.yaml | 2 +- examples/etcd/etcd-iochaos.yaml | 1 - install.sh | 19 ++++ ...car-template.yaml => chaosfs-sidecar.yaml} | 3 +- pkg/webhook/config/config_test.go | 2 +- test/action.go | 2 +- test/pkg/fixture/fixture.go | 2 +- 13 files changed, 200 insertions(+), 44 deletions(-) create mode 100644 docs/template_config.md rename manifests/{sidecar-template.yaml => chaosfs-sidecar.yaml} (96%) diff --git a/docs/io_chaos.md b/docs/io_chaos.md index 72c7240d1c..bd76581546 100644 --- a/docs/io_chaos.md +++ b/docs/io_chaos.md @@ -29,6 +29,19 @@ IOChaos needs to inject a sidecar container to user pods and the sidecar contain > * While admission controllers are enabled by default, some Kubernetes distributions might disable them. In this case, follow the instructions to [turn on admission controllers](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#how-do-i-turn-on-an-admission-controller). > * [ValidatingAdmissionWebhooks](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook) and [MutatingAdmissionWebhooks](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#mutatingadmissionwebhook) are required by IOChaos. +### Template Configuration + +Chaos Mesh uses a template mechanism to simplify the configuration of sidecar injection. + +Because of the `Go Template` conflict with `Helm`, the common template is not included in the [Helm Chart](../helm/chaos-mesh). +However, it will be deployed automatically if you install Chaos Mesh via the [install script](../install.sh). + +By default, the common template ConfigMaps should be deployed in the same namespace as Chaos Mesh. + +```bash +kubectl apply -f manifests/chaosfs-sidecar.yaml -n +``` + ### Data directory The data directory of the application in the target pod should be a **subdirectory** of `PersistentVolumes`. @@ -56,11 +69,11 @@ ARGS="--pd=${CLUSTER_NAME}-pd:2379 \ > * If you are testing a TiDB cluster, you need to modify it at [`_start_tikv.sh.tpl`](https://github.com/pingcap/tidb-operator/blob/master/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl). > * PD has the same issue with TiKV. You need to modify the data directory of PD at [`_start_pd.sh.tpl`](https://github.com/pingcap/tidb-operator/blob/master/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl). -## ConfigMap configuration +## Injection configuration -Chaos Mesh uses sidecar container to inject IOChaos. To fulfill this chaos, you need to configure this sidecar container using a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/). +Injection configuration is another [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) and is required to fulfill IO Chaos. -To define a specified ConfigMap for your application before starting your chaos experiment, refer to this [document](sidecar_configmap.md). +To define a specified ConfigMap for your application before starting your chaos experiment, please refer to this [document](./sidecar_configmap.md). You can apply the ConfigMap defined for your application to Kubernetes cluster by the following command: @@ -82,7 +95,6 @@ spec: action: mixed mode: one duration: "400s" - configName: "chaosfs-tikv" path: "" selector: labelSelectors: @@ -108,27 +120,22 @@ For more sample files, see [examples](../examples). You can edit them as needed. | **path** | Defines the path of files for injecting IOChaos actions. It should be a regular expression for the path which you want to inject errno or delay. If the path is `""` or not defined, the IOChaos action is injected into all files.| | | **methods** | Defines the IO methods for injecting IOChaos actions. It is an array of string, which sets the IO syscalls. | `open` / `read` See the [available methods](#available-methods) for more details. | | **addr** | Defines the sidecar HTTP server address for a sidecar container.| `":8080"` | -| **configName** | Defines the configuration name which is used to inject chaos action into pods. You can refer to [examples/tikv-configmap.yaml](../examples/chaosfs-configmap/tikv-configmap.yaml) to define your configuration.| | | **layer** | Represents the layer of the IO action.| `fs` (by default). | ## Usage -Before the application created, you need to make admission-webhook enabled using labels and [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) to the application namespace: +Before the application created, you need to enable admission-webhook enabled on the application namespace: ```bash -admission-webhook.pingcap.com/init-request:chaosfs-tikv +kubectl create ns app-ns +kubectl label ns app-ns admission-webhook=enabled ``` -You can use the following commands to set labels and annotations of the application namespace: +Then we have two ways to mark the pods we want to inject IO Chaos: -```bash -# If the application namespace does not exist. you can exec this command to create one, -# otherwise ignore this command. -kubectl create ns app-ns # "app-ns" is the application namespace - -# enable admission-webhook -kubectl label ns app-ns admission-webhook=enabled +1. Set annotation `admission-webhook.pingcap.com/init-request` on the namespace, then all pods in this namespace meet the selector requirements will be injected. +```bash # set annotation kubectl annotate ns app-ns admission-webhook.pingcap.com/init-request=chaosfs-tikv @@ -136,8 +143,14 @@ kubectl annotate ns app-ns admission-webhook.pingcap.com/init-request=chaosfs-ti ... ``` +2. Set annotation `admission-webhook.pingcap.com/request` on the pods, you can check this [example](../examples/etcd/etcd.yaml). + Then, you can start your application and define YAML file to start your chaos experiment. +> **Note:** +> +> The value of the annotation in the above examples, `chaos-tikv` is the name filed in your injection config. + ### Start a chaos experiment Assume that you are using `examples/io-mixed-example.yaml`, you can run the following command to create a chaos experiment: diff --git a/docs/sidecar_configmap.md b/docs/sidecar_configmap.md index fa338bd89c..9c7a4b74e5 100644 --- a/docs/sidecar_configmap.md +++ b/docs/sidecar_configmap.md @@ -8,11 +8,19 @@ Chaos Mesh runs a [fuse-daemon](https://www.kernel.org/doc/Documentation/filesys In sidecar container, fuse-daemon needs to mount the data directory of application by [fusermount](http://manpages.ubuntu.com/manpages/bionic/en/man1/fusermount.1.html) before the application starts. -Most applications use different data directories, so you need to define the different sidecar configs for different applications. +## How it works? -## Configuration file +Currently, Chaos Mesh supports two types of ConfigMaps: -The following content is the sidecar ConfigMap defined for tikv: +1. Template config. The skeleton of each sidecar config is similar, in order to fulfill different requirements and make the configuration simplified, +Chaos Mesh supports creating common templates to be used by different applications. For the details of template configuration, please refer to [template config](./template_config.md). + +2. Injection config. This configuration will be combined with template config and finally generate a config to inject to the selected pods. +Since most applications use different data directories, volume name or container name, you can define different parameters based on the common template created in the first step. + +## Injection Configuration + +The following content is an injection ConfigMap defined for tikv: ```yaml --- @@ -24,21 +32,46 @@ metadata: labels: app.kubernetes.io/component: webhook data: - # the real content of config - chaosfs-tikv.yaml: | + chaosfs-tikv: | name: chaosfs-tikv selector: labelSelectors: "app.kubernetes.io/component": "tikv" + template: chaosfs-sidecar + arguments: + ContainerName: "tikv" + DataPath: "/var/lib/tikv/data" + MountPath: "/var/lib/tikv" + VolumeName: "tikv" +``` + +Injection config defines some injection arguments for different applications, and it is based on the common template created beforehand. + +For fields defined in this config, we have some brief descriptions below: + +* **name**: injection config name, uniquely identifies a injection config in one namespace. + However, you can have the same name in different namespaces so this is useful to implement multi-tenancy. +* **selector**: is used to filter pods to inject sidecar. +* **template**: the template config map name used to render the injection config. +* **arguments**: the arguments you should define to be used in the template. + +The final injection config content is rendered by `template` and `arguments` via `Go Template` and +will be injected to the selected pods. In this example, the final injection config is: + +``` + name: chaosfs-tikv + selector: + labelSelectors: + "app.kubernetes.io/component": "tikv" initContainers: - name: inject-scripts image: pingcap/chaos-scripts:latest - imagePullpolicy: Always + imagePullPolicy: Always command: ["sh", "-c", "/scripts/init.sh -d /var/lib/tikv/data -f /var/lib/tikv/fuse-data"] containers: - name: chaosfs image: pingcap/chaos-fs:latest - imagePullpolicy: Always + imagePullPolicy: Always ports: - containerPort: 65534 securityContext: @@ -72,19 +105,7 @@ data: - /tmp/scripts/wait-fuse.sh ``` -For more sample ConfigMap files, see [examples](https://github.com/pingcap/chaos-mesh/tree/master/examples/chaosfs-configmap). - -Description: - -* **name**: defines the name of the sidecar config, this name should be unique across all sidecar configs. -* **selector**: is used to filter pods to inject sidecar. -* **initContainers**: defines the [initContainer](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) need to be injected. -* **container**: defines the sidecar container need to be injected. -* **volumeMounts**: defines the new volumeMounts or overwrite the old volumeMounts of the each containers in target pods. -* **volume**: defines the new volumes for the target pod or overwrite the old volumes in target pods. -* **postStart**: called after a container is created first. If the handler fails, the containers will failed. - -Key defines for the name of deployment container. Value defines for the Commands for stating container. +For more sample ConfigMap files, see [examples](../examples/chaosfs-configmap). ### Containers diff --git a/docs/template_config.md b/docs/template_config.md new file mode 100644 index 0000000000..0b517467ae --- /dev/null +++ b/docs/template_config.md @@ -0,0 +1,103 @@ +# Template Config + +The following content is the common template ConfigMap defined for injecting IO Chaos sidecar, you can also find this example [here](../manifests/chaosfs-sidecar.yaml): + +```yaml +--- +apiVersion: v0 +kind: ConfigMap +metadata: + name: chaosfs-sidecar + labels: + app.kubernetes.io/component: template +data: + data: | + initContainers: + - name: inject-scripts + image: pingcap/chaos-scripts:latest + imagePullPolicy: Always + command: ["sh", "-c", "/scripts/init.sh -d {{.DataPath}} -f {{.MountPath}}/fuse-data"] + containers: + - name: chaosfs + image: pingcap/chaos-fs:latest + imagePullPolicy: Always + ports: + - containerPort: 65533 + securityContext: + privileged: true + command: + - /usr/local/bin/chaosfs + - -addr=:65533 + - -pidfile=/tmp/fuse/pid + - -original={{.MountPath}}/fuse-data + - -mountpoint={{.DataPath}} + volumeMounts: + - name: {{.VolumeName}} + mountPath: {{.MountPath}} + mountPropagation: Bidirectional + volumeMounts: + - name: {{.VolumeName}} + mountPath: {{.MountPath}} + mountPropagation: HostToContainer + - name: scripts + mountPath: /tmp/scripts + - name: fuse + mountPath: /tmp/fuse + volumes: + - name: scripts + emptyDir: {} + - name: fuse + emptyDir: {} + postStart: + {{.ContainerName}}: + command: + - /tmp/scripts/wait-fuse.sh +``` + +Template config defines some variables by [Go Template](https://golang.org/pkg/text/template/) mechanism. This example has four arguments: + +- DataPath: original data directory +- MountPath: after injecting chaosfs sidecar, data directory will be mounted to {{.MountPath}}/fuse-data +- VolumeName: the data volume name used by the pod +- ContainerName: to which container the sidecar is injected + +For fields defined in this template, we have some brief descriptions below: + +* **initContainers**: defines the [initContainer](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) need to be injected. +* **container**: defines the sidecar container need to be injected. +* **volumeMounts**: defines the new volumeMounts or overwrite the old volumeMounts of each containers in target pods. +* **volume**: defines the new volumes for the target pod or overwrite the old volumes in target pods. +* **postStart**: called after a container is created first. If the handler fails, the containers will failed. + +> **Note:** +> +> Chaos controller-manager only watches template config map with the label selector specified by its flag `--template-labels`, by default this label +> is `app.kubernetes.io/component=template` if your Chaos Mesh is deployed by Helm. +> +> Each template config map should be deployed in the same namespace as Chaos Mesh, and it is identified by the name of the config map, which is `chaosfs-sidecar` in the above example. +> +> The template config content should be in the `data` field. This means it is not possible to define two templates in one config map, you have to use two config maps like the example below. + +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: chaosfs-sidecar0 + labels: + app.kubernetes.io/component: template +data: + data: | + xxxx + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: chaosfs-sidecar1 + labels: + app.kubernetes.io/component: template +data: + data: | + xxxx +``` diff --git a/examples/chaosfs-configmap/pd-configmap.yaml b/examples/chaosfs-configmap/pd-configmap.yaml index 8c54e06ed3..6b3e924dfc 100644 --- a/examples/chaosfs-configmap/pd-configmap.yaml +++ b/examples/chaosfs-configmap/pd-configmap.yaml @@ -12,7 +12,7 @@ data: selector: labelSelectors: "app.kubernetes.io/component": "pd" - template: sidecar-template + template: chaosfs-sidecar arguments: ContainerName: "pd" DataPath: "/var/lib/pd/data" diff --git a/examples/chaosfs-configmap/tiflash-configmap.yaml b/examples/chaosfs-configmap/tiflash-configmap.yaml index c76ffcc9ae..6e3c747e2d 100644 --- a/examples/chaosfs-configmap/tiflash-configmap.yaml +++ b/examples/chaosfs-configmap/tiflash-configmap.yaml @@ -12,7 +12,7 @@ data: selector: labelSelectors: "app.kubernetes.io/component": "tiflash" - template: sidecar-template + template: chaosfs-sidecar arguments: ContainerName: "pd" DataPath: "/data/db" diff --git a/examples/chaosfs-configmap/tikv-configmap.yaml b/examples/chaosfs-configmap/tikv-configmap.yaml index d957099363..5dba236368 100644 --- a/examples/chaosfs-configmap/tikv-configmap.yaml +++ b/examples/chaosfs-configmap/tikv-configmap.yaml @@ -12,7 +12,7 @@ data: selector: labelSelectors: "app.kubernetes.io/component": "tikv" - template: sidecar-template + template: chaosfs-sidecar arguments: ContainerName: "tikv" DataPath: "/var/lib/tikv/data" diff --git a/examples/etcd/etcd-configmap.yaml b/examples/etcd/etcd-configmap.yaml index 58223efbee..150ada61e5 100644 --- a/examples/etcd/etcd-configmap.yaml +++ b/examples/etcd/etcd-configmap.yaml @@ -12,7 +12,7 @@ data: selector: labelSelectors: app: etcd - template: sidecar-template + template: chaosfs-sidecar arguments: ContainerName: "etcd" DataPath: "/var/run/etcd/default.etcd" diff --git a/examples/etcd/etcd-iochaos.yaml b/examples/etcd/etcd-iochaos.yaml index 8ed890a44a..854c09bd8d 100644 --- a/examples/etcd/etcd-iochaos.yaml +++ b/examples/etcd/etcd-iochaos.yaml @@ -6,7 +6,6 @@ spec: action: delay mode: one duration: "400s" - configName: "chaosfs-etcd" path: "" selector: labelSelectors: diff --git a/install.sh b/install.sh index 7426bfba1d..e4ab754799 100755 --- a/install.sh +++ b/install.sh @@ -73,6 +73,7 @@ main() { local crd="https://raw.githubusercontent.com/pingcap/chaos-mesh/master/manifests/crd.yaml" local runtime="docker" local template=false + local sidecar_template=true while [[ $# -gt 0 ]] do @@ -176,6 +177,11 @@ main() { shift shift ;; + --sidecar_template) + sidecar_template=true + shift + shift + ;; *) echo "unknown flag or option $key" usage @@ -201,6 +207,9 @@ main() { if $template; then ensure gen_crd_manifests "${crd}" ensure gen_chaos_mesh_manifests "${runtime}" + if $sidecar_template; then + ensure gen_default_sidecar_template + fi exit 0 fi @@ -569,6 +578,7 @@ install_chaos_mesh() { local docker_mirror=$5 local crd=$6 local runtime=$7 + local sidecar_template=$8 printf "Install Chaos Mesh %s\n" "${release_name}" @@ -586,6 +596,9 @@ install_chaos_mesh() { gen_crd_manifests "${crd}" | kubectl apply -f - || exit 1 gen_chaos_mesh_manifests "${runtime}" | kubectl apply -f - || exit 1 + if [ "$sidecar_template" == "true" ]; then + gen_default_sidecar_template | kubectl apply -f - || exit 1 + fi } version_lt() { @@ -756,6 +769,12 @@ gen_crd_manifests() { fi } +gen_default_sidecar_template() { + local template_path="manifests/chaosfs-sidecar.yaml" + + ensure cat "$template_path" +} + gen_chaos_mesh_manifests() { local runtime=$1 local socketPath="/var/run/docker.sock" diff --git a/manifests/sidecar-template.yaml b/manifests/chaosfs-sidecar.yaml similarity index 96% rename from manifests/sidecar-template.yaml rename to manifests/chaosfs-sidecar.yaml index 0343aac00f..fb9a047574 100644 --- a/manifests/sidecar-template.yaml +++ b/manifests/chaosfs-sidecar.yaml @@ -2,7 +2,8 @@ apiVersion: v1 kind: ConfigMap metadata: - name: sidecar-template + name: chaosfs-sidecar + namespace: chaos-testing labels: app.kubernetes.io/component: template data: diff --git a/pkg/webhook/config/config_test.go b/pkg/webhook/config/config_test.go index 3392d146ee..f26a1feaef 100644 --- a/pkg/webhook/config/config_test.go +++ b/pkg/webhook/config/config_test.go @@ -27,7 +27,7 @@ name: chaosfs-etcd selector: labelSelectors: app: etcd -template: sidecar-template +template: chaosfs-sidecar arguments: ContainerName: "etcd" DataPath: "/var/run/etcd/default.etcd" diff --git a/test/action.go b/test/action.go index 4a7c640c67..0f51657039 100644 --- a/test/action.go +++ b/test/action.go @@ -164,7 +164,7 @@ func (oa *operatorAction) InstallTemplate(info OperatorConfig) error { templateCM := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: info.Namespace, - Name: "sidecar-template", + Name: "chaosfs-sidecar", Labels: map[string]string{ "app.kubernetes.io/component": "template", }, diff --git a/test/pkg/fixture/fixture.go b/test/pkg/fixture/fixture.go index 55a36c3ee2..a2d668c48c 100644 --- a/test/pkg/fixture/fixture.go +++ b/test/pkg/fixture/fixture.go @@ -29,7 +29,7 @@ const ioTestConfigMap = `name: chaosfs-io selector: labelSelectors: app: io -template: sidecar-template +template: chaosfs-sidecar arguments: ContainerName: "io" DataPath: "/var/run/data/test"