2.5.0 nfs (dummy)

.travis.yml

@@ -2,7 +2,7 @@ sudo: required
 dist: xenial
 language: go
 go:
-  - 1.18.3
+  - 1.19.1
 before_install:
   - sudo apt-get update -yq || true
   - sudo apt-get install go-md2man -y
@@ -12,16 +12,16 @@ cache:
     - $HOME/.cache/go-build
 script:
   - |
-    if [ "${TRAVIS_BRANCH}" == "master" ]; then
-      export DOCKER_HUB_STORK_TAG=master
-      export DOCKER_HUB_STORK_TEST_TAG=latest
-      export DOCKER_HUB_CMD_EXECUTOR_TAG=master
+    if [ "${TRAVIS_BRANCH}" == "2.5.0-nfs" ]; then
+      export DOCKER_HUB_STORK_TAG="${TRAVIS_BRANCH}"-dev
+      export DOCKER_HUB_STORK_TEST_TAG="${TRAVIS_BRANCH}"-dev
+      export DOCKER_HUB_CMD_EXECUTOR_TAG="${TRAVIS_BRANCH}"-dev
     else
       export DOCKER_HUB_STORK_TAG=`git rev-parse --short HEAD`
       export DOCKER_HUB_STORK_TEST_TAG=`git rev-parse --short HEAD`
       export DOCKER_HUB_CMD_EXECUTOR_TAG=`git rev-parse --short HEAD`
     fi
-    travis_wait 30 make -j 2 && make test && make container && make integration-test && make integration-test-container &&
+    travis_wait 45 make -j 2 && make test && make container && make integration-test && make integration-test-container &&
     if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
       docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}";
       make deploy;

Dockerfile

@@ -18,16 +18,19 @@ RUN python3 -m pip install awscli  && python3 -m pip install rsa --upgrade
 RUN curl -q -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator && \
     chmod +x /usr/local/bin/aws-iam-authenticator
 
-ARG GCLOUD_SDK=google-cloud-sdk-413.0.0-linux-x86_64.tar.gz
-# Remove the test directories
-# Also don't need gsutil
+#Install Google Cloud SDK
+ARG GCLOUD_SDK=google-cloud-sdk-418.0.0-linux-x86_64.tar.gz
+ARG GCLOUD_INSTALL_DIR="/usr/lib"
 RUN curl -q -o $GCLOUD_SDK https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/$GCLOUD_SDK && \
-    tar xf $GCLOUD_SDK && rm -rf $GCLOUD_SDK && \
-    rm -rf /google-cloud-sdk/platform/gsutil/third_party/oauth2client/tests \
-        /google-cloud-sdk/platform/gsutil/third_party/rsa/tests \
-        /google-cloud-sdk/platform/gsutil/third_party/httplib2/python2/httplib2/test \
-        /google-cloud-sdk/platform/gsutil && \
-    python3 -m pip install pyyaml>=5.1 rsa>=4.0 urllib3>=1.24.2 --upgrade -t /google-cloud-sdk/lib/third_party
+    tar xf $GCLOUD_SDK -C $GCLOUD_INSTALL_DIR && rm -rf $GCLOUD_SDK && \
+    rm -rf $GCLOUD_INSTALL_DIR/google-cloud-sdk/platform/gsutil \
+           $GCLOUD_INSTALL_DIR/google-cloud-sdk/RELEASE_NOTES
+ENV PATH "${PATH}:$GCLOUD_INSTALL_DIR/google-cloud-sdk/bin"
+#Install gke-gcloud-auth-plugin
+RUN gcloud components install gke-gcloud-auth-plugin
+#Create symlink /google-cloud-sdk/bin -> /usr/lib/google-cloud-sdk/bin for legacy cluster pair with gcp auth plugin
+RUN mkdir google-cloud-sdk
+RUN ln -s /usr/lib/google-cloud-sdk/bin /google-cloud-sdk/bin
 
 WORKDIR /
 

Makefile

@@ -31,7 +31,7 @@ ifeq ($(BUILD_TYPE),debug)
 BUILDFLAGS += -gcflags "-N -l"
 endif
 
-RELEASE_VER := 2.7.0
+RELEASE_VER := 2.5.0-nfs
 BASE_DIR    := $(shell git rev-parse --show-toplevel)
 GIT_SHA     := $(shell git rev-parse --short HEAD)
 BIN         :=$(BASE_DIR)/bin
@@ -113,7 +113,9 @@ test:
 
 integration-test:
 	@echo "Building stork integration tests"
-	@cd test/integration_test && GOOS=linux go test -tags integrationtest $(BUILD_OPTIONS) -v -c -o stork.test
+	docker run --rm -v $(shell pwd):/go/src/github.com/libopenstorage/stork  $(DOCK_BUILD_CNT) \
+		   /bin/bash -c 'cd /go/src/github.com/libopenstorage/stork/test/integration_test && \
+		   GOOS=linux go test -tags integrationtest $(BUILD_OPTIONS) -v -c -o stork.test;'
 
 integration-test-container:
 	@echo "Building container: docker build --tag $(STORK_TEST_IMG) -f Dockerfile ."
@@ -130,21 +132,30 @@ codegen:
 
 stork:
 	@echo "Building the stork binary"
-	@cd cmd/stork && CGO_ENABLED=0 GOOS=linux go build $(BUILD_OPTIONS) -o $(BIN)/stork
+	docker run --rm -v $(shell pwd):/go/src/github.com/libopenstorage/stork  $(DOCK_BUILD_CNT) \
+           /bin/bash -c 'cd /go/src/github.com/libopenstorage/stork/cmd/stork && \
+		   CGO_ENABLED=0 GOOS=linux go build $(BUILD_OPTIONS) -o /go/src/github.com/libopenstorage/stork/bin/stork;'
 
 cmdexecutor:
 	@echo "Building command executor binary"
-	@cd cmd/cmdexecutor && GOOS=linux go build $(BUILD_OPTIONS) -o $(BIN)/cmdexecutor
+	docker run --rm -v $(shell pwd):/go/src/github.com/libopenstorage/stork  $(DOCK_BUILD_CNT) \
+		/bin/bash -c 'cd /go/src/github.com/libopenstorage/stork/cmd/cmdexecutor && \
+		GOOS=linux go build $(BUILD_OPTIONS) -o /go/src/github.com/libopenstorage/stork/bin/cmdexecutor;'
 
 storkctl:
 	@echo "Building storkctl"
-	@cd cmd/storkctl && CGO_ENABLED=0 GOOS=linux go build $(BUILD_OPTIONS) -o $(BIN)/linux/storkctl
-	@cd cmd/storkctl && CGO_ENABLED=0 GOOS=darwin go build $(BUILD_OPTIONS) -o $(BIN)/darwin/storkctl
-	@cd cmd/storkctl && CGO_ENABLED=0 GOOS=windows go build $(BUILD_OPTIONS) -o $(BIN)/windows/storkctl.exe
+	docker run --rm -v $(shell pwd):/go/src/github.com/libopenstorage/stork  $(DOCK_BUILD_CNT) \
+		/bin/bash -c 'cd /go/src/github.com/libopenstorage/stork/cmd/storkctl; \
+		CGO_ENABLED=0 GOOS=linux go build $(BUILD_OPTIONS) -o /go/src/github.com/libopenstorage/stork/bin/linux/storkctl; \
+		CGO_ENABLED=0 GOOS=darwin go build $(BUILD_OPTIONS) -o /go/src/github.com/libopenstorage/stork/bin/darwin/storkctl; \
+		CGO_ENABLED=0 GOOS=windows go build $(BUILD_OPTIONS) -o /go/src/github.com/libopenstorage/stork/bin/windows/storkctl.exe;'
 
 px-statfs:
 	@echo "Building px_statfs.so"
-	@cd drivers/volume/portworx/px-statfs && gcc -g -shared -fPIC -o $(BIN)/px_statfs.so px_statfs.c -ldl -D__USE_LARGEFILE64
+	docker run --rm -v $(shell pwd):/go/src/github.com/libopenstorage/stork  $(DOCK_BUILD_CNT) \
+		   /bin/bash -c 'cd /go/src/github.com/libopenstorage/stork/drivers/volume/portworx/px-statfs &&  \
+           gcc -g -shared -fPIC -o /go/src/github.com/libopenstorage/stork/bin/px_statfs.so px_statfs.c -ldl -D__USE_LARGEFILE64;'
+
 
 container: help
 	@echo "Building container: docker build --build-arg VERSION=$(DOCKER_HUB_STORK_TAG) --build-arg RELEASE=$(DOCKER_HUB_STORK_TAG) --tag $(STORK_IMG) -f Dockerfile . "

cmd/stork/stork.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"flag"
-	"github.com/libopenstorage/stork/pkg/cache"
 	"net/http"
 	"os"
 	"os/signal"
@@ -23,6 +22,7 @@ import (
 	_ "github.com/libopenstorage/stork/drivers/volume/portworx"
 	"github.com/libopenstorage/stork/pkg/apis"
 	"github.com/libopenstorage/stork/pkg/applicationmanager"
+	"github.com/libopenstorage/stork/pkg/cache"
 	"github.com/libopenstorage/stork/pkg/clusterdomains"
 	"github.com/libopenstorage/stork/pkg/dbg"
 	"github.com/libopenstorage/stork/pkg/extender"
@@ -43,6 +43,7 @@ import (
 	"github.com/libopenstorage/stork/pkg/webhookadmission"
 	kdmpapi "github.com/portworx/kdmp/pkg/apis/kdmp/v1alpha1"
 	"github.com/portworx/kdmp/pkg/controllers/dataexport"
+	"github.com/portworx/kdmp/pkg/controllers/resourceexport"
 	"github.com/portworx/kdmp/pkg/drivers"
 	"github.com/portworx/kdmp/pkg/jobratelimit"
 	kdmpversion "github.com/portworx/kdmp/pkg/version"
@@ -187,12 +188,12 @@ func main() {
 		cli.IntFlag{
 			Name:  "k8s-api-qps",
 			Value: 100,
-			Usage: "Restrict number of k8s api requests from stork (default: 100 QPS)",
+			Usage: "Restrict number of k8s API requests from stork (default: 100 QPS)",
 		},
 		cli.IntFlag{
 			Name:  "k8s-api-burst",
 			Value: 100,
-			Usage: "Restrict number of k8s api requests from stork (default: 100 Burst)",
+			Usage: "Restrict number of k8s API requests from stork (default: 100 Burst)",
 		},
 		cli.BoolTFlag{
 			Name:  "kdmp-controller",
@@ -209,6 +210,10 @@ func main() {
 		},
 	}
 
+	// Export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+	log.Warnf("Export USE_GKE_GCLOUD_AUTH_PLUGIN=True")
+	os.Setenv("USE_GKE_GCLOUD_AUTH_PLUGIN", "True")
+
 	if err := app.Run(os.Args); err != nil {
 		log.Fatalf("Error starting stork: %v", err)
 	}
@@ -393,7 +398,7 @@ func run(c *cli.Context) {
 			EventRecorder: recorder,
 		}
 		resourceLock, err := resourcelock.New(
-			resourcelock.ConfigMapsResourceLock,
+			resourcelock.ConfigMapsLeasesResourceLock,
 			lockObjectNamespace,
 			lockObjectName,
 			k8sClient.CoreV1(),
@@ -547,6 +552,14 @@ func runStork(mgr manager.Manager, ctx context.Context, d volume.Driver, recorde
 		if err := dataexport.Init(mgr); err != nil {
 			log.Fatalf("Error initializing kdmp controller: %v", err)
 		}
+		resourceexport, err := resourceexport.NewController(mgr)
+		if err != nil {
+			log.Fatalf("Error initializing resource export controller: %v", err)
+		}
+
+		if err := resourceexport.Init(mgr); err != nil {
+			log.Fatalf("Error initializing resource export controller manager: %v", err)
+		}
 	}
 
 	go func() {

drivers/volume/aws/aws.go

@@ -116,11 +116,17 @@ func (a *aws) OwnsPVCForBackup(
 	pvc *v1.PersistentVolumeClaim,
 	cmBackupType string,
 	crBackupType string,
+	blType storkapi.BackupLocationType,
 ) bool {
 	if cmBackupType == storkapi.ApplicationBackupGeneric {
 		// If user has forced the backupType in config map, default to generic always
 		return false
 	}
+	// For AWS volume and backuplocation type is NFS, we will not own.
+	// It will default to kdmp
+	if blType == storkapi.BackupLocationNFS {
+		return false
+	}
 	return a.OwnsPVC(coreOps, pvc)
 }
 

drivers/volume/azure/azure.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"time"
 
@@ -127,7 +127,7 @@ func (a *azure) getMetadata() (map[string]string, error) {
 	if resp.StatusCode != 200 {
 		return nil, fmt.Errorf("error querying Azure metadata: Code %d returned for url %s", resp.StatusCode, req.URL)
 	}
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, fmt.Errorf("error querying Azure metadata: %v", err)
 	}
@@ -157,11 +157,17 @@ func (a *azure) OwnsPVCForBackup(
 	pvc *v1.PersistentVolumeClaim,
 	cmBackupType string,
 	crBackupType string,
+	blType storkapi.BackupLocationType,
 ) bool {
 	if cmBackupType == storkapi.ApplicationBackupGeneric {
 		// If user has forced the backupType in config map, default to generic always
 		return false
 	}
+	// For Azure based volume and backuplocation type is NFS, we will not own.
+	// It will default to kdmp
+	if blType == storkapi.BackupLocationNFS {
+		return false
+	}
 	return a.OwnsPVC(coreOps, pvc)
 }