Golang v1.11 module support for build-tools (#60)

* Bump to golang v1.11.4
* Convert to go modules
* Add the with_vendor directory to the test
* Update tests for go 1.11 behavior
* Use -mod=vendor if vendor directory exists

.buildkite/test.sh

@@ -2,20 +2,15 @@
 
 # This script is used to build drud/build-tools using buildkite
 
-# Manufacture a $GOPATH environment that can mount on docker (when buildkite build)
-export GOPATH=~/tmp/buildkite-fake-gopath/$BUILDKITE_JOB_ID
-DRUDSRC=$GOPATH/src/github.com/drud
-mkdir -p $DRUDSRC
-ln -s $PWD $DRUDSRC/build-tools
-cd $DRUDSRC/build-tools
-BUILD_OS=$(go env GOOS)
-echo "--- buildkite building $BUILDKITE_JOB_ID at $(date) on $HOSTNAME for OS=$(go env GOOS) in $PWD GOPATH=$GOPATH"
 
 set -o errexit
 set -o pipefail
 set -o nounset
 set -x
 
+BUILD_OS=$(go env GOOS)
+echo "--- buildkite building $BUILDKITE_JOB_ID at $(date) on $HOSTNAME for OS=$(go env GOOS) in $PWD"
+
 # Our testbot should now be sane, run the testbot checker to make sure.
 echo "--- Checking for sane testbot"
 ./.buildkite/sanetestbot.sh

.circleci/config.yml

@@ -3,18 +3,15 @@ stages:
   build:
     machine:
       image: circleci/classic:201711-01
-    working_directory: /home/circleci/go/src/github.com/drud/build-tools
 
     environment:
       GOPATH: /home/circleci/go
 
     steps:
       - run: sudo rm -rf /usr/local/go &&
-             wget -q -O /tmp/golang.tgz https://dl.google.com/go/go1.11.linux-amd64.tar.gz &&
+             wget -q -O /tmp/golang.tgz https://dl.google.com/go/go1.11.4.linux-amd64.tar.gz &&
              sudo tar -C /usr/local -xzf /tmp/golang.tgz
 
-      - run: mkdir -p ~/go/src/github.com/drud/build-tools && mkdir -p ~/go/lib && mkdir -p ~/go/bin
-
       - checkout
 
       - run: cd tests && make test

README.md

@@ -40,10 +40,9 @@ make linux
 make darwin
 make gofmt
 make govet
-make govendor
 make golint
 make codecoroner
-make static (gofmt, govet, golint, govendor)
+make static (gofmt, govet, golint)
 make test
 make container
 make push

makefile_components/base_build_go.mak

@@ -4,8 +4,21 @@
 ##### contents into ../Makefile and commenting out the include and adding a
 ##### comment about what you did and why.
 
-
-.PHONY: all build test push clean container-clean bin-clean version static govendor gofmt govet golint golangci-lint container
+DOCKERBUILDCMD=docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
+          	    -v "$(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)"                                \
+          	    -v "$(S)$$PWD:/workdir$(DOCKERMOUNTFLAG)"                              \
+          	    -e CGO_ENABLED=0                  \
+          	    -e GOOS=$@						  \
+          	    -w $(S)/workdir              \
+          	    $(BUILD_IMAGE)
+
+DOCKERTESTCMD=docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
+          	    -v "$(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)"                                \
+          	    -v "$(S)$$PWD:/workdir$(DOCKERMOUNTFLAG)"                              \
+          	    -w $(S)/workdir              \
+          	    $(BUILD_IMAGE)
+
+.PHONY: all build test push clean container-clean bin-clean version static gofmt govet golint golangci-lint container
 GOTMP=.gotmp
 
 SHELL = /bin/bash
@@ -14,7 +27,7 @@ GOFILES = $(shell find $(SRC_DIRS) -name "*.go")
 
 BUILD_OS = $(shell go env GOHOSTOS)
 
-BUILD_IMAGE ?= drud/golang-build-container:v1.11
+BUILD_IMAGE ?= drud/golang-build-container:v1.11.4.2
 
 BUILD_BASE_DIR ?= $$PWD
 
@@ -35,6 +48,11 @@ VERSION_LDFLAGS := $(foreach v,$(VERSION_VARIABLES),-X "$(PKG)/pkg/version.$(v)=
 LDFLAGS := -extldflags -static $(VERSION_LDFLAGS)
 DOCKERMOUNTFLAG := :delegated
 
+# In go 1.11 -mod=vendor is not autodetected; it probably will be in 1.12
+# See https://github.com/golang/go/issues/27227
+USEMODVENDOR := $(shell if [ -d vendor ]; then echo "-mod=vendor"; fi)
+
+
 PWD=$(shell pwd)
 S =
 ifeq ($(BUILD_OS),windows)
@@ -47,143 +65,70 @@ build: $(BUILD_OS)
 
 linux darwin windows: $(GOFILES)
 	@echo "building $@ from $(SRC_AND_UNDER)"
-	@$(shell rm -f VERSION.txt)
-	@$(shell mkdir -p bin/$@ $(GOTMP)/{std/$@,bin,src/$(PKG)})
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
-	    -v "$(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)"                                \
-	    -v "$(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)"                              \
-	    -v "$(S)$$PWD/bin/$@:/go/bin$(DOCKERMOUNTFLAG)"                        \
-	    -v "$(S)$$PWD/bin/$@:/go/bin/$@$(DOCKERMOUNTFLAG)"             \
-	    -v "$(S)$$PWD/$(GOTMP)/std/$@:/usr/local/go/pkg/$@_amd64_static$(DOCKERMOUNTFLAG)"  \
-	    -e CGO_ENABLED=0                  \
-	    -e GOOS=$@						  \
-	    -w $(S)/go/src/$(PKG)                 \
-	    $(BUILD_IMAGE)                    \
-        go install -installsuffix static -ldflags ' $(LDFLAGS) ' $(SRC_AND_UNDER)
+	@mkdir -p $(GOTMP)/{.cache,pkg,src,bin}
+	@$(DOCKERBUILDCMD) \
+        go install $(USEMODVENDOR) -installsuffix static -ldflags ' $(LDFLAGS) ' $(SRC_AND_UNDER)
 	@$(shell touch $@)
+	$( shell if [ -d $(GOTMP) ]; then chmod -R u+w $(GOTMP); fi )
 	@echo $(VERSION) >VERSION.txt
 
-govendor:
-	@echo -n "Using govendor to check for missing dependencies and unused dependencies: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                   \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                  \
-		-w $(S)/go/src/$(PKG)                                         \
-		$(BUILD_IMAGE)                                                     \
-		bash -c 'OUT=$$(govendor list +missing +unused); if [ -n "$$OUT" ]; then echo "$$OUT"; exit 1; fi'
-
 gofmt:
 	@echo "Checking gofmt: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		bash -c 'export OUT=$$(gofmt -l $(SRC_DIRS))  && if [ -n "$$OUT" ]; then echo "These files need gofmt -w: $$OUT"; exit 1; fi'
 
 govet:
 	@echo "Checking go vet: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $S/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		bash -c 'go vet $(SRC_AND_UNDER)'
 
 golint:
 	@echo "Checking golint: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                   \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		bash -c 'export OUT=$$(golint $(SRC_AND_UNDER)) && if [ -n "$$OUT" ]; then echo "Golint problems discovered: $$OUT"; exit 1; fi'
 
 errcheck:
 	@echo "Checking errcheck: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                   \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		errcheck $(SRC_AND_UNDER)
 
 staticcheck:
 	@echo "Checking staticcheck: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		staticcheck $(SRC_AND_UNDER)
 
-unused:
-	@echo "Checking unused variables and functions: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
-		unused $(SRC_AND_UNDER)
-
-codecoroner:
-	@echo "Checking codecoroner for unused functions: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE) \
-		bash -c 'OUT=$$(codecoroner -tests -ignore vendor funcs $(SRC_AND_UNDER)); if [ -n "$$OUT" ]; then echo "$$OUT"; exit 1; fi'                                             \
-
-
 varcheck:
 	@echo "Checking unused globals and struct members: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
-		varcheck $(SRC_AND_UNDER) && structcheck $(SRC_AND_UNDER)
+	@$(DOCKERTESTCMD) \
+		bash -c "varcheck $(SRC_AND_UNDER) && structcheck $(SRC_AND_UNDER)"
 
 misspell:
 	@echo "Checking for misspellings: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		misspell $(SRC_DIRS)
 
 gometalinter:
 	@echo "gometalinter: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                         \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	@$(DOCKERTESTCMD) \
 		time gometalinter $(GOMETALINTER_ARGS) $(SRC_AND_UNDER)
 
 golangci-lint:
 	@echo "golangci-lint: "
-	@docker run -t --rm -u $(shell id -u):$(shell id -g)                       \
-		-v $(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)                            \
-		-v $(S)$$PWD:/go/src/$(PKG)$(DOCKERMOUNTFLAG)                          \
-		-w $(S)/go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                         \
+	@$(DOCKERTESTCMD) \
 		time bash -c "golangci-lint run $(GOLANGCI_LINT_ARGS) $(SRC_AND_UNDER)"
 
 version:
 	@echo VERSION:$(VERSION)
 
 clean: container-clean bin-clean
-	@go clean -cache || echo "You're not running latest golang locally" # Make sure the local go cache is clean for testing
 
 container-clean:
 	@if docker image inspect $(DOCKER_REPO):$(VERSION) >/dev/null 2>&1; then docker rmi -f $(DOCKER_REPO):$(VERSION); fi
 	@rm -rf .container-* .dockerfile* .push-* linux darwin windows container VERSION.txt .docker_image
 
 bin-clean:
-	$(shell rm -rf $(GOTMP) bin .tmp)
+	@rm -rf bin
+	$(shell if [ -d $(GOTMP) ]; then chmod -R u+w $(GOTMP) && rm -rf $(GOTMP); fi )
 
 # print-ANYVAR prints the expanded variable
 print-%: ; @echo $* = $($*)

makefile_components/base_test_go.mak

@@ -7,9 +7,8 @@
 TESTOS = $(BUILD_OS)
 
 test: build
-	@mkdir -p bin/linux
-	@mkdir -p $(GOTMP)/{src/$(PKG),pkg,bin,std/linux}
 	@echo "Testing $(SRC_AND_UNDER) with TESTARGS=$(TESTARGS)"
+	@mkdir -p $(GOTMP)/{.cache,pkg,src,bin}
 	@docker run -t --rm  -u $(shell id -u):$(shell id -g)                 \
 	    -v $(PWD)/$(GOTMP):/go$(DOCKERMOUNTFLAG)                                                 \
 	    -v $(PWD):/go/src/$(PKG)$(DOCKERMOUNTFLAG)                                          \
@@ -18,7 +17,8 @@ test: build
 	    -e CGO_ENABLED=0	\
 	    -w /go/src/$(PKG)                                                  \
 	    $(BUILD_IMAGE)                                                     \
-        go test -v -installsuffix static -ldflags '$(LDFLAGS)' $(SRC_AND_UNDER) $(TESTARGS)
+        go test $(USEMODVENDOR) -v -installsuffix static -ldflags '$(LDFLAGS)' $(SRC_AND_UNDER) $(TESTARGS)
+	@$(shell chmod -R u+w $(GOTMP))
 
 # test_precompile allows a full compilation of _test.go files, without execution of the tests.
 # Setup and teardown in TestMain is still executed though, so this can cost some time.

tests/Makefile

@@ -13,6 +13,14 @@ UPSTREAM_REPO ?= golang:1.7.5-alpine3.5
 
 export WORKING_DIR = $(shell pwd)
 
+TESTDOCKERCMD=docker run -t --rm -u $(shell id -u):$(shell id -g)                    \
+          	    -v "$(S)$$PWD/$(GOTMP):/go$(DOCKERMOUNTFLAG)"                                \
+          	    -v "$(S)$$PWD:/workdir$(DOCKERMOUNTFLAG)"                              \
+          	    -v "$(S)$$PWD/$(GOTMP)/std/$@:/usr/local/go/pkg/$@_amd64_static$(DOCKERMOUNTFLAG)"  \
+          	    -w $(S)/workdir              \
+          	    $(BUILD_IMAGE)
+
+
 # Top-level directories to build - Only build the explicit complex fail stuff by default. Unit test will be overridden.
 SRC_DIRS := cmd pkg
 
@@ -54,10 +62,9 @@ include ../makefile_components/base_push.mak
 # We can't use the standard 'test' target because this one actually uses 'make' and counts on resources unavailable in
 # golang compiler container.
 test: build
-	@mkdir -p bin/linux
-	@mkdir -p  $(GOTMP)/{src/$(PKG),pkg,bin,std/linux}
-	go test -v -installsuffix "static" -ldflags '$(LDFLAGS)' $(SRC_AND_UNDER) $(TESTARGS)
-	$(MAKE) -C standard_target $@
+	@go test $(USEMODVENDOR) -v -installsuffix "static" -ldflags '$(LDFLAGS)' $(SRC_AND_UNDER) $(TESTARGS)
+	@$(MAKE) -C standard_target $@
+
 
 # test_precompile allows a full compilation of _test.go files, without execution of the tests.
 # Setup and teardown in TestMain is still executed though, so this can cost some time.
@@ -66,15 +73,9 @@ test_precompile: test
 
 # Simple way to execute a random command in the container for tests - used only for testing
 # Example: make COMMAND="govendor fetch golang.org/x/net/context"
-COMMAND := govendor list
+# COMMAND := govendor list
 container_cmd:
-	@docker run                                                            \
-		-t --rm -u $(shell id -u):$(shell id -g)                         \
-	    -v $(PWD)/$(GOTMP):/go                                                 \
-	    -v $(PWD):/go/src/$(PKG)                                          \
-	    -v $(PWD)/bin/linux:/go/bin                                     \
-	    -v $(PWD)/$(GOTMP)/std/linux:/usr/local/go/pkg/linux_amd64_static  \
-		-w /go/src/$(PKG)                                                  \
-		$(BUILD_IMAGE)                                                     \
+	#@$(shell if [ -f ./go.mod ] ; then GOPATH=$$PWD/$(GOTMP) go get ./...;  fi; )
+	@$(TESTDOCKERCMD) \
 		$(COMMAND)
 

tests/go.mod

@@ -0,0 +1,12 @@
+module github.com/drud/build-tools/tests
+
+require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/dominikh/go-tools v0.0.0-20190102075043-fe93b0e3b36b // indirect
+	github.com/lextoumbourou/goodhosts v2.1.0+incompatible
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1
+	golang.org/x/tools v0.0.0-20190102183724-79186431cf29 // indirect
+	honnef.co/go/tools v0.0.0-20190102075043-fe93b0e3b36b // indirect
+)

tests/go.sum

@@ -0,0 +1,16 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dominikh/go-tools v0.0.0-20190102075043-fe93b0e3b36b h1:cjKkva93RDqb43YJD5xikcP47FR4r9U9O2Sd40tiBTU=
+github.com/dominikh/go-tools v0.0.0-20190102075043-fe93b0e3b36b/go.mod h1:qBNLV2EQXN9hMDGPA+nlUNDKcPCtmEvCC8iVgA6uBic=
+github.com/lextoumbourou/goodhosts v2.1.0+incompatible h1:1U1p5Z1wrXl23/fW/GY4zdTbQ8UJbyvrkPbqAZ6tzbw=
+github.com/lextoumbourou/goodhosts v2.1.0+incompatible/go.mod h1:89s48k108X3gKDWn8AHk3gUzUGTcMZCCAOsE4QU1bbo=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1 h1:Zx8Rp9ozC4FPFxfEKRSUu8+Ay3sZxEUZ7JrCWMbGgvE=
+github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/tools v0.0.0-20190102183724-79186431cf29 h1:zIIZ0Uf4ZEDJmZrJJKpW8g8N2+p3o0v0a9L44AJxAVY=
+golang.org/x/tools v0.0.0-20190102183724-79186431cf29/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+honnef.co/go/tools v0.0.0-20190102075043-fe93b0e3b36b h1:81sBftiEKRto8JWPj1APoVShwNrPxLja6VpjobUNXxs=
+honnef.co/go/tools v0.0.0-20190102075043-fe93b0e3b36b/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=