diff --git a/deploy/kubernetes/base/ss-csi-linode-controller.yaml b/deploy/kubernetes/base/ss-csi-linode-controller.yaml index 9a3e88ff..9f8219ee 100644 --- a/deploy/kubernetes/base/ss-csi-linode-controller.yaml +++ b/deploy/kubernetes/base/ss-csi-linode-controller.yaml @@ -21,12 +21,13 @@ spec: initContainers: - name: init image: bitnami/kubectl:1.16.3-debian-10-r36 - command: + command: - /scripts/get-linode-id.sh env: - name: NODE_NAME valueFrom: fieldRef: + apiVersion: v1 fieldPath: spec.nodeName volumeMounts: - name: linode-info @@ -35,8 +36,10 @@ spec: mountPath: /scripts containers: - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v1.6.0 + image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0 + imagePullPolicy: IfNotPresent args: + - "--default-fstype=ext4" - "--volume-name-prefix=pvc" - "--volume-name-uuid-length=16" - "--csi-address=$(ADDRESS)" @@ -44,24 +47,24 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "Always" volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v2.2.0 + image: k8s.gcr.io/sig-storage/csi-attacher:v3.3.0 + imagePullPolicy: IfNotPresent args: - "--v=2" - "--csi-address=$(ADDRESS)" env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - imagePullPolicy: "Always" volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - - name: linode-csi-resizer - image: quay.io/k8scsi/csi-resizer:v0.5.0 + - name: csi-resizer + image: k8s.gcr.io/sig-storage/csi-resizer:v1.3.0 + imagePullPolicy: IfNotPresent args: - "--v=2" - "--csi-address=$(ADDRESS)" @@ -73,7 +76,7 @@ spec: mountPath: /var/lib/csi/sockets/pluginproxy/ - name: linode-csi-plugin image: linode/linode-blockstorage-csi-driver:latest - args : + args: - "--endpoint=$(CSI_ENDPOINT)" - "--token=$(LINODE_TOKEN)" - "--url=$(LINODE_API_URL)" @@ -86,17 +89,16 @@ spec: - name: LINODE_API_URL value: https://api.linode.com/v4 - name: LINODE_BS_PREFIX - value: - name: NODE_NAME valueFrom: fieldRef: + apiVersion: v1 fieldPath: spec.nodeName - name: LINODE_TOKEN valueFrom: secretKeyRef: name: linode key: token - imagePullPolicy: "Always" volumeMounts: - name: linode-info mountPath: /linode-info @@ -104,6 +106,13 @@ spec: mountPath: /scripts - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists volumes: - name: socket-dir emptyDir: {} diff --git a/e2e/test/csi_driver_test.go b/e2e/test/csi_driver_test.go index 71d4512c..68336929 100644 --- a/e2e/test/csi_driver_test.go +++ b/e2e/test/csi_driver_test.go @@ -2,178 +2,265 @@ package test import ( "e2e_test/test/framework" + "fmt" "strconv" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var _ = Describe("CSIDriver", func() { - var ( - err error - pod *core.Pod - pvc *core.PersistentVolumeClaim - f *framework.Invocation - size string - file = "/data/heredoc" - storageClass = "linode-block-storage" - ) - BeforeEach(func() { - f = root.Invoke() - }) +var _ = Describe("Linode CSI Driver", func() { + Describe("A StatefulSet with a PVC", func() { + Context("Using VolumeClaimTemplates and a non-root container", func() { + var ( + f *framework.Invocation + storageClass = "linode-block-storage" + sts *appsv1.StatefulSet + pod *core.Pod + pvc *core.PersistentVolumeClaim + file = "/data/file.txt" + ) - var writeFile = func(filename string) { - By("Writing a File into the Pod") - err = f.WriteFileIntoPod(filename, pod) - Expect(err).NotTo(HaveOccurred()) - } - - var readFile = func(filename string) { - By("Checking the Created File into the Pod") - err = f.CheckFileIntoPod(filename, pod) - Expect(err).NotTo(HaveOccurred()) - } - - var expandVolume = func(size string) { - By("Expanding Size of the Persistent Volume") - currentPVC, err := f.GetPersistentVolumeClaim(pvc.ObjectMeta) - Expect(err).NotTo(HaveOccurred()) - - currentPVC.Spec.Resources.Requests = core.ResourceList{ - core.ResourceName(core.ResourceStorage): resource.MustParse(size), - } - err = f.UpdatePersistentVolumeClaim(currentPVC) - Expect(err).NotTo(HaveOccurred()) - - By("Checking if Volume Expansion Occurred") - Eventually(func() string { - s, _ := f.GetVolumeSize(currentPVC) - return strconv.Itoa(s) + "Gi" - }, f.Timeout, f.RetryInterval).Should(Equal(size)) - } - - Describe("Test", func() { - Context("Simple", func() { - Context("Block Storage", func() { - JustBeforeEach(func() { - By("Creating Persistent Volume Claim") - pvc = f.GetPersistentVolumeClaimObject(size, storageClass) - err = f.CreatePersistentVolumeClaim(pvc) - Expect(err).NotTo(HaveOccurred()) - - By("Creating Pod with PVC") - pod = f.GetPodObject(pvc.Name) - err = f.CreatePod(pod) - Expect(err).NotTo(HaveOccurred()) - }) + BeforeEach(func() { + f = root.Invoke() + By("Getting the StatefulSet manifest w/ non-root container") + sts = framework.GetStatefulSetObject("redis-test", f.Namespace(), storageClass) - AfterEach(func() { - By("Deleting the Pod with PVC") - err = f.DeletePod(pod.ObjectMeta) - Expect(err).NotTo(HaveOccurred()) - - By("Getting the Volume information") - currentPVC, err := f.GetPersistentVolumeClaim(pvc.ObjectMeta) - Expect(err).NotTo(HaveOccurred()) - volumeID, err := f.GetVolumeID(currentPVC) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for the Volume to be Detached") - Eventually(func() bool { - isAttached, err := f.IsVolumeDetached(volumeID) - if err != nil { - return false - } - return isAttached - }, f.Timeout, f.RetryInterval).Should(BeTrue()) - - By("Deleting the PVC") - err = f.DeletePersistentVolumeClaim(pvc.ObjectMeta) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for the Volume to be Deleted") - Eventually(func() bool { - isDeleted, err := f.IsVolumeDeleted(volumeID) - if err != nil { - return false - } - return isDeleted - }, f.Timeout, f.RetryInterval).Should(BeTrue()) - }) + By("Creating the StatefulSet in the cluster") + Eventually(func() error { + return f.CreateStatefulSet(sts) + }, f.Timeout, f.RetryInterval).Should(Succeed()) - Context("1Gi Storage", func() { - BeforeEach(func() { - size = "1Gi" - }) - It("should write and read", func() { - writeFile(file) - readFile(file) - }) - }) + By("Waiting until the StatefulSet Pod is healthy") + Eventually(func() error { + var err error + pod, err = f.GetPod("redis-test-0", f.Namespace()) + if err != nil { + return err + } + return f.WaitForReady(pod.ObjectMeta) + }, f.Timeout, f.RetryInterval).Should(Succeed()) + + By("Checking that there is a PVC created for the StatefulSet") + Eventually(func() error { + var err error + pvc, err = f.GetPersistentVolumeClaim(metav1.ObjectMeta{Name: "data-redis-test-0", Namespace: f.Namespace()}) + if err != nil { + return err + } + return nil + }, f.Timeout, f.RetryInterval).Should(Succeed()) + }) + + AfterEach(func() { + By("Deleting the StatefulSet") + Eventually(func() error { + return f.DeleteStatefulSet(sts.ObjectMeta) + }, f.Timeout, f.RetryInterval).Should(Succeed()) + + By("Getting the Volume information") + var ( + currentPVC *core.PersistentVolumeClaim + volumeID int + ) + Eventually(func() error { + var err error + currentPVC, err = f.GetPersistentVolumeClaim(pvc.ObjectMeta) + if err != nil { + return err + } + volumeID, err = f.GetVolumeID(currentPVC) + if err != nil { + return err + } + return nil + }, f.Timeout, f.RetryInterval).Should(Succeed()) + + By("Waiting for the Volume to be Detached") + Eventually(func() bool { + isAttached, err := f.IsVolumeDetached(volumeID) + if err != nil { + return false + } + return isAttached + }, f.Timeout, f.RetryInterval).Should(BeTrue()) + + By("Deleting the PVC") + Eventually(func() error { + return f.DeletePersistentVolumeClaim(pvc.ObjectMeta) + }, f.Timeout, f.RetryInterval).Should(Succeed()) + + By("Waiting for the Volume to be Deleted") + Eventually(func() bool { + isDeleted, err := f.IsVolumeDeleted(volumeID) + if err != nil { + return false + } + return isDeleted + }, f.Timeout, f.RetryInterval).Should(BeTrue()) + }) + + It("Ensures no data is lost between Pod deletions", func() { + var err error + By("Saving a file in the mounted directory within the container") + err = f.WriteFileIntoPod(file, pod) + Expect(err).NotTo(HaveOccurred()) + + By("Deleting the StatefulSet Pod") + Eventually(func() error { + return f.DeletePod(pod.ObjectMeta) + }, f.Timeout, f.RetryInterval).Should(Succeed()) + + By("Waiting until the StatefulSet Pod is recreated") + Eventually(func() error { + name := "redis-test-0" + p, err := f.GetPod(name, f.Namespace()) + if err != nil { + return err + } + if p.ObjectMeta.UID == pod.ObjectMeta.UID { + return fmt.Errorf("pod %s/%s not deleted", f.Namespace(), name) + } + pod = p + return nil + }, f.Timeout, f.RetryInterval).Should(Succeed()) + Eventually(func() error { + return f.WaitForReady(pod.ObjectMeta) + }, f.Timeout, f.RetryInterval).Should(Succeed()) + + By("Checking that the file is still present inside the container") + err = f.CheckIfFileIsInPod(file, pod) + Expect(err).NotTo(HaveOccurred()) }) }) }) - Describe("Test", func() { - Context("Block Storage", func() { - Context("Volume Expansion", func() { - JustBeforeEach(func() { - By("Creating Persistent Volume Claim") - pvc = f.GetPersistentVolumeClaimObject(size, storageClass) - err = f.CreatePersistentVolumeClaim(pvc) - Expect(err).NotTo(HaveOccurred()) - - By("Creating Pod with PVC") - pod = f.GetPodObject(pvc.Name) - err = f.CreatePod(pod) - Expect(err).NotTo(HaveOccurred()) - }) + Describe("A Pod with a PVC", func() { + var ( + f *framework.Invocation + pod *core.Pod + pvc *core.PersistentVolumeClaim + size string + file = "/data/heredoc" + storageClass = "linode-block-storage" + ) + + var writeFile = func(filename string) { + By("Writing a file into the Pod") + err := f.WriteFileIntoPod(filename, pod) + Expect(err).NotTo(HaveOccurred()) + } + + var readFile = func(filename string) { + By("Checking if the created file is in the Pod") + err := f.CheckIfFileIsInPod(filename, pod) + Expect(err).NotTo(HaveOccurred()) + } + + var expandVolume = func(size string) { + By("Expanding size of the Persistent Volume") + currentPVC, err := f.GetPersistentVolumeClaim(pvc.ObjectMeta) + Expect(err).NotTo(HaveOccurred()) + + currentPVC.Spec.Resources.Requests = core.ResourceList{ + core.ResourceName(core.ResourceStorage): resource.MustParse(size), + } + err = f.UpdatePersistentVolumeClaim(currentPVC) + Expect(err).NotTo(HaveOccurred()) + + By("Checking if Volume expansion occurred") + Eventually(func() string { + s, _ := f.GetVolumeSize(currentPVC) + return strconv.Itoa(s) + "Gi" + }, f.Timeout, f.RetryInterval).Should(Equal(size)) + } + + Context("Using a Pod with a PVC mounted", func() { + JustBeforeEach(func() { + f = root.Invoke() + By("Creating the Persistent Volume Claim") + pvc = framework.GetPersistentVolumeClaimObject("test-pvc", f.Namespace(), size, storageClass) + Eventually(func() error { + return f.CreatePersistentVolumeClaim(pvc) + }, f.Timeout, f.RetryInterval).Should(Succeed()) + + By("Creating Pod with PVC") + pod = framework.GetPodObject("busybox-test", f.Namespace(), pvc.Name) + Eventually(func() error { + return f.CreatePod(pod) + }, f.Timeout, f.RetryInterval).Should(Succeed()) + }) - AfterEach(func() { - By("Deleting the Pod with PVC") - err = f.DeletePod(pod.ObjectMeta) - Expect(err).NotTo(HaveOccurred()) - - By("Getting the Volume information") - currentPVC, err := f.GetPersistentVolumeClaim(pvc.ObjectMeta) - Expect(err).NotTo(HaveOccurred()) - volumeID, err := f.GetVolumeID(currentPVC) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for the Volume to be Detached") - Eventually(func() bool { - isDetached, err := f.IsVolumeDetached(volumeID) - if err != nil { - return false - } - return isDetached - }, f.Timeout, f.RetryInterval).Should(BeTrue()) - - By("Deleting the PVC") - err = f.DeletePersistentVolumeClaim(pvc.ObjectMeta) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for the Volume to be Deleted") - Eventually(func() bool { - isDeleted, err := f.IsVolumeDeleted(volumeID) - if err != nil { - return false - } - return isDeleted - }, f.Timeout, f.RetryInterval).Should(BeTrue()) + AfterEach(func() { + By("Deleting the Pod with PVC") + Eventually(func() error { + return f.DeletePod(pod.ObjectMeta) + }, f.Timeout, f.RetryInterval).Should(Succeed()) + + By("Getting the Volume information") + var ( + currentPVC *core.PersistentVolumeClaim + volumeID int + ) + Eventually(func() error { + var err error + currentPVC, err = f.GetPersistentVolumeClaim(pvc.ObjectMeta) + if err != nil { + return err + } + volumeID, err = f.GetVolumeID(currentPVC) + if err != nil { + return err + } + return nil + }, f.Timeout, f.RetryInterval).Should(Succeed()) + + By("Waiting for the Volume to be Detached") + Eventually(func() bool { + isAttached, err := f.IsVolumeDetached(volumeID) + if err != nil { + return false + } + return isAttached + }, f.Timeout, f.RetryInterval).Should(BeTrue()) + + By("Deleting the PVC") + Eventually(func() error { + return f.DeletePersistentVolumeClaim(pvc.ObjectMeta) + }, f.Timeout, f.RetryInterval).Should(Succeed()) + + By("Waiting for the Volume to be Deleted") + Eventually(func() bool { + isDeleted, err := f.IsVolumeDeleted(volumeID) + if err != nil { + return false + } + return isDeleted + }, f.Timeout, f.RetryInterval).Should(BeTrue()) + }) + + Context("1Gi Storage", func() { + BeforeEach(func() { + size = "1Gi" + }) + It("should write and read", func() { + writeFile(file) + readFile(file) }) + }) - Context("Expanding Storage from 10Gi to 15Gi", func() { - BeforeEach(func() { - size = "10Gi" - }) - It("should write and read", func() { - writeFile(file) - expandVolume("15Gi") - readFile(file) - }) + Context("Expanding Storage from 10Gi to 15Gi", func() { + BeforeEach(func() { + size = "10Gi" + }) + It("should write and read", func() { + writeFile(file) + expandVolume("15Gi") + readFile(file) }) }) }) diff --git a/e2e/test/e2e_suite_test.go b/e2e/test/e2e_suite_test.go index 5c5da38a..c68ed648 100644 --- a/e2e/test/e2e_suite_test.go +++ b/e2e/test/e2e_suite_test.go @@ -113,7 +113,7 @@ var _ = AfterSuite(func() { } By("Deleting Namespace " + root.Namespace()) - err := root.DeleteNamespace() + err := root.DeleteNamespace(root.Namespace()) Expect(err).NotTo(HaveOccurred()) if !(useExisting || reuse) { By("Deleting cluster") diff --git a/e2e/test/framework/namespace.go b/e2e/test/framework/namespace.go index a6799b29..c5114297 100644 --- a/e2e/test/framework/namespace.go +++ b/e2e/test/framework/namespace.go @@ -19,6 +19,6 @@ func (f *Framework) CreateNamespace() error { return err } -func (f *Framework) DeleteNamespace() error { - return f.kubeClient.CoreV1().Namespaces().Delete(f.namespace, deleteInForeground()) +func (f *Framework) DeleteNamespace(name string) error { + return f.kubeClient.CoreV1().Namespaces().Delete(name, deleteInForeground()) } diff --git a/e2e/test/framework/pod.go b/e2e/test/framework/pod.go index b0d330bc..046343a4 100644 --- a/e2e/test/framework/pod.go +++ b/e2e/test/framework/pod.go @@ -2,6 +2,7 @@ package framework import ( "fmt" + "github.com/appscode/go/wait" "github.com/pkg/errors" core "k8s.io/api/core/v1" @@ -9,16 +10,16 @@ import ( "kmodules.xyz/client-go/tools/exec" ) -func (f *Invocation) GetPodObject(pvc string) *core.Pod { +func GetPodObject(name, namespace, pvc string) *core.Pod { return &core.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: f.app, - Namespace: f.namespace, + Name: name, + Namespace: namespace, }, Spec: core.PodSpec{ Containers: []core.Container{ { - Name: f.app, + Name: name, Image: "busybox", VolumeMounts: []core.VolumeMount{ { @@ -44,7 +45,7 @@ func (f *Invocation) GetPodObject(pvc string) *core.Pod { } func (f *Invocation) CreatePod(pod *core.Pod) error { - pod, err := f.kubeClient.CoreV1().Pods(f.namespace).Create(pod) + pod, err := f.kubeClient.CoreV1().Pods(pod.ObjectMeta.Namespace).Create(pod) if err != nil { return err } @@ -53,16 +54,16 @@ func (f *Invocation) CreatePod(pod *core.Pod) error { } func (f *Invocation) DeletePod(meta metav1.ObjectMeta) error { - return f.kubeClient.CoreV1().Pods(f.namespace).Delete(meta.Name, deleteInForeground()) + return f.kubeClient.CoreV1().Pods(meta.Namespace).Delete(meta.Name, deleteInForeground()) } -func (f *Invocation) GetPod(name, ns string) (*core.Pod, error) { - return f.kubeClient.CoreV1().Pods(ns).Get(name, metav1.GetOptions{}) +func (f *Invocation) GetPod(name, namespace string) (*core.Pod, error) { + return f.kubeClient.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) } func (f *Invocation) WaitForReady(meta metav1.ObjectMeta) error { return wait.PollImmediate(f.RetryInterval, f.Timeout, func() (bool, error) { - pod, err := f.kubeClient.CoreV1().Pods(f.namespace).Get(meta.Name, metav1.GetOptions{}) + pod, err := f.kubeClient.CoreV1().Pods(meta.Namespace).Get(meta.Name, metav1.GetOptions{}) if pod == nil || err != nil { return false, nil } @@ -81,7 +82,7 @@ func (f *Invocation) WriteFileIntoPod(filename string, pod *core.Pod) error { return err } -func (f *Invocation) CheckFileIntoPod(filename string, pod *core.Pod) error { +func (f *Invocation) CheckIfFileIsInPod(filename string, pod *core.Pod) error { out, err := exec.ExecIntoPod(f.restConfig, pod, exec.Command([]string{ "ls", filename, }...)) diff --git a/e2e/test/framework/pvc.go b/e2e/test/framework/pvc.go index ce6bb393..4ba921b3 100644 --- a/e2e/test/framework/pvc.go +++ b/e2e/test/framework/pvc.go @@ -11,11 +11,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (f *Invocation) GetPersistentVolumeClaimObject(size, storageClass string) *core.PersistentVolumeClaim { +func GetPersistentVolumeClaimObject(name, namespace, size, storageClass string) *core.PersistentVolumeClaim { return &core.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: f.app, - Namespace: f.namespace, + Name: name, + Namespace: namespace, }, Spec: core.PersistentVolumeClaimSpec{ AccessModes: []core.PersistentVolumeAccessMode{ diff --git a/e2e/test/framework/statefulset.go b/e2e/test/framework/statefulset.go new file mode 100644 index 00000000..30fa537e --- /dev/null +++ b/e2e/test/framework/statefulset.go @@ -0,0 +1,96 @@ +package framework + +import ( + appsv1 "k8s.io/api/apps/v1" + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func GetStatefulSetObject(name, namespace, storageClass string) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": name, + "app.kubernetes.io/instance": name, + }, + }, + Spec: appsv1.StatefulSetSpec{ + ServiceName: name, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": name, + }, + }, + Template: core.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "app.kubernetes.io/name": name, + }, + }, + Spec: core.PodSpec{ + SecurityContext: &core.PodSecurityContext{ + FSGroup: func(i int64) *int64 { return &i }(1001), + }, + AutomountServiceAccountToken: func(b bool) *bool { return &b }(false), + Containers: []core.Container{ + { + Name: name, + Image: "bitnami/redis", + Env: []core.EnvVar{ + { + Name: "ALLOW_EMPTY_PASSWORD", + Value: "true", + }, + }, + SecurityContext: &core.SecurityContext{ + RunAsUser: func(i int64) *int64 { return &i }(1001), + }, + VolumeMounts: []core.VolumeMount{ + { + Name: "data", + MountPath: "/data", + }, + }, + }, + }, + }, + }, + VolumeClaimTemplates: []core.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{Name: "data"}, + Spec: core.PersistentVolumeClaimSpec{ + StorageClassName: &storageClass, + AccessModes: []core.PersistentVolumeAccessMode{ + core.ReadWriteOnce, + }, + Resources: core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceName(core.ResourceStorage): resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + } +} + +func (f *Invocation) CreateStatefulSet(sts *appsv1.StatefulSet) error { + _, err := f.kubeClient.AppsV1().StatefulSets(sts.ObjectMeta.Namespace).Create(sts) + if err != nil { + return err + } + return nil +} + +func (f *Invocation) DeleteStatefulSet(meta metav1.ObjectMeta) error { + return f.kubeClient.AppsV1().StatefulSets(meta.Namespace).Delete(meta.Name, deleteInForeground()) +} + +func (f *Invocation) GetStatefulSet(name, namespace string) (*appsv1.StatefulSet, error) { + return f.kubeClient.AppsV1().StatefulSets(namespace).Get(name, metav1.GetOptions{}) +}