Skip to content

Commit

Permalink
Merge commit '28e4e26f004d03e3447acabfdda2d20e8bfe5f0c'
Browse files Browse the repository at this point in the history
  • Loading branch information
Flynn committed Sep 24, 2019
2 parents a498bd7 + 28e4e26 commit 2470958
Show file tree
Hide file tree
Showing 8 changed files with 526 additions and 499 deletions.
2 changes: 2 additions & 0 deletions build-aux/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
<!-- -*- fill-column: 100 -*- -->
# Datawire build-aux CHANGELOG

- 2019-09-12: `docker.mk`: Drop the built-in build rule; is now strictly bring-your-own-build-rule.

- 2019-08-14: `go-mod.mk`: `go.bins`, `go.list`: Correctly prune nested Go modules in git
submodules.

Expand Down
5 changes: 1 addition & 4 deletions build-aux/bin-go/kubeapply/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,4 @@ module github.com/datawire/build-aux/bin-go/kubeapply

go 1.12

require github.com/datawire/teleproxy v0.3.16

// Fix invalid pseudo-version that Go 1.13 complains about.
replace github.com/census-instrumentation/opencensus-proto v0.1.0-0.20181214143942-ba49f56771b8 => github.com/census-instrumentation/opencensus-proto v0.0.3-0.20181214143942-ba49f56771b8
require github.com/datawire/teleproxy v0.7.3-0.20190913173620-2f688be1248e
368 changes: 170 additions & 198 deletions build-aux/bin-go/kubeapply/go.sum

Large diffs are not rendered by default.

5 changes: 1 addition & 4 deletions build-aux/bin-go/teleproxy/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,4 @@ module github.com/datawire/build-aux/bin-go/teleproxy

go 1.12

require github.com/datawire/teleproxy v0.3.16

// Fix invalid pseudo-version that Go 1.13 complains about.
replace github.com/census-instrumentation/opencensus-proto v0.1.0-0.20181214143942-ba49f56771b8 => github.com/census-instrumentation/opencensus-proto v0.0.3-0.20181214143942-ba49f56771b8
require github.com/datawire/teleproxy v0.7.3-0.20190919160255-4ad7074eeea1
369 changes: 174 additions & 195 deletions build-aux/bin-go/teleproxy/go.sum

Large diffs are not rendered by default.

272 changes: 176 additions & 96 deletions build-aux/docker.mk
Original file line number Diff line number Diff line change
Expand Up @@ -5,106 +5,136 @@
## Eager inputs ##
# - Variables : docker.tag.$(GROUP) # define %.docker.tag.$(GROUP) and %.docker.push.$(GROUP) targets
## Lazy inputs ##
# (none)
# - Target: : $(NAME).docker # build untagged image; define this for each image $(NAME)
## Outputs ##
#
# - Executable : WRITE_DOCKERTAGFILE ?= $(CURDIR)/build-aux/bin/write-dockertagfile
#
# - Variable : HAVE_DOCKER # non-empty if true, empty if false
# - Variable : docker.LOCALHOST # "host.docker.internal" on Docker for Desktop, "localhost" on Docker CE
#
# - Target : %.docker: %/Dockerfile # build image (untagged)
# - Target : %.docker.tag.$(GROUP) # tag image as $(docker.tag.$(GROUP))
# - Target : %.docker.push.$(GROUP) # push tag(s) $(docker.tag.$(GROUP))
# - .PHONY Target : %.docker.clean # remove image and tags
# - Target : %.docker.tag.$(GROUP) # tag image as $(docker.tag.$(GROUP))
# - Target : %.docker.push.$(GROUP) # push tag(s) $(docker.tag.$(GROUP))
# - .PHONY Target : %.docker.clean # remove image and tags
#
## common.mk targets ##
# (none)
#
# ## Local docker build ##
#
# To use this Makefile snippet naively, `Dockerfile`s must be in
# sub-directories; it doesn't support out-of-the-box having a
# `Dockerfile` in the root. If you would like to have
# `Dockerfile`s out of the root (or with other names, like
# `Dockerfile.base-envoy`), then you must supply your own
# `%.docker` target that
# 1. Calls `docker build --iidfile=TEMPFILE ...`
# 2. Calls `$(MOVE_IFCHANGED) TEMPFILE $@`
#
# You can build a Docker image by depending on `SOMEPATH.docker`.
# If you provide a custom `%.docker` rule, then of course exactly
# what that builds will be different, but with the default built-in
# rule: depending on `SOMEPATH.docker` will build
# `SOMEPATH/Dockefile`. This will build the image, but NOT tag it
# (see below for tagging).
#
# You can untag and remove an image by having your `clean` target
# depend on `SOMEPATH.docker.clean`.
#
# With the default built-in rule:
#
# - If you need something to be done before the `docker build`,
# make it a dependency of `SOMEPATH.docker`.
#
# - If you need something (`FILE`) to be included in the build
# context, copy it to `SOMEPATH/` by having
# `SOMEPATH.docker` depend on `SOMEPATH/FILE`.
#
# ## Working with those untagged images ##
#
# - Tagging: You can tag an image after being built by depending
# on `SOMEPATH.docker.tag.GROUP`, where you've set up GROUP by
# writing
#
# docker.tag.GROUP = EXPR
#
# _before_ including docker.mk, where GROUP is the suffix of the
# target that you'd like to depend on in your Makefile, and EXPR
# is a Makefile expression that evaluates to 1 or more tag
# names; it is evaluated in the context of
# `SOMEPATH.docker.tag.GROUP`; specifically:
# * `$*` is set to SOMEPATH
# * `$<` is set to a file containing the image ID
#
# Additionally, you can override the EXPR on a per-image basis
# by overriding the `docker.tag.GROUP` variable on a per-target
# basis:
#
# SOMEPATH.docker.tag.GROUP: docker.tag.GROUP = EXPR
#
# > For example:
# >
# > docker.tag.release = quay.io/datawire/ambassador_pro:$(notdir $*)-$(VERSION)
# > docker.tag.buildcache = quay.io/datawire/ambassador_pro-buildcache:$(notdir $*)-$(VERSION)
# > include build-aux/docker.mk
# > # The above will cause docker.mk to define targets:
# > # - %.docker.tag.release
# > # - %.docker.push.release
# > # - %.docker.tag.buildcache
# > # - %.docker.push.buildcache
# >
# > # Override the release name a specific image.
# > # Release ambassador-withlicense/ambassador.docker
# > # - based on the above : quay.io/datawire/ambassador_pro:ambassador-$(VERSION)
# > # - after being overridden: quay.io/datawire/ambassador_pro:amb-core-$(VERSION)
# > ambassador-withlicense/ambassador.docker.tag.release: docker.tag.release = quay.io/datawire/ambassador_pro:amb-core-$(VERSION)
#
# - Pushing a tag: You can push tags that have been created with
# `SOMEPATH.docker.tag.GROUP` (see above) by depending on
# `SOMEPATH.docker.push.GROUP`.
#
# > For example:
# > The Ambassador Pro images:
# > - get built from: `docker/$(NAME)/Dockerfile`
# > - get pushed as : `quay.io/datawire/ambassador_pro:$(NAME)-$(VERSION)`
# >
# > We accomplish this by saying:
# >
# > docker.tag.release = quay.io/datawire/ambassador_pro:$(notdir $*)-$(VERSION)
# >
# > and having our `build` target depend on `NAME.docker.tag.release` (for each NAME)
# > and having our `release` target depend on `NAME.docker.push.release` (for each NAME)
# This Makefile snippet helps you manage Docker images as files from a
# Makefile. Think of it as glue. It doesn't dictate how to build
# your images or what flags you pass to `docker build`--you must
# provide your own rule that calls `docker build`. It doesn't dictate
# how your image tags are named. It provides glue to keep track of
# those image, and flexibly but coherently push them to any of
# multiple remote Docker repositories. All while being careful to not
# leave dangling images in your Docker cache that force you to run
# `docker image prune` an unreasonable amount.
#
# ## Building ##
#
# For each Docker IMAGE you would like to build, you need to
# provide your own build-rule for `IMAGE.docker`. There are 2
# requirements for the rule:
#
# 1. It must write a file named `IMAGE.docker` (for your value of
# IMAGE) containing just the Image ID. (This is trivially
# accomplished using the --iidfile argument to `docker build`.)
#
# 2. It must only adjust the timestamp of IMAGE.docker if the
# contents of the file change. (This is trivially accomplished
# using any of the `*_IFCHANGED` build-aux helper programs.)
#
# The simplest version of that is:
#
# IMAGE.docker: $(MOVE_IFCHANGED) FORCE
# docker build [email protected] .
# $(MOVE_IFCHANGED) [email protected] $@
#
# If you have multiple `Dockerfile` at `IMAGE/Dockerfile`, you
# might write a pattern rule:
#
# %.docker: %/Dockerfile $(MOVE_IFCHANGED) FORCE
# docker build [email protected] $*
# $(MOVE_IFCHANGED) [email protected] $@
#
# Unless you have a good reason to, you shouldn't concern yourself
# with tagging the image in this rule.
#
# See the "More build-rule examples" section below for more
# examples.
#
# ## Tagging ##
#
# You can tag an image after being built by depending on
# `IMAGE.docker.tag.GROUP`, where you've set up GROUP by writing
#
# docker.tag.GROUP = EXPR
#
# _before_ including `docker.mk`, where GROUP is the suffix of the
# target that you'd like to depend on in your Makefile, and EXPR is
# a Makefile expression that evaluates to one-or-more tag names; it
# is evaluated in the context of `IMAGE.docker.tag.GROUP`;
# specifically:
#
# * `$*` is set to IMAGE
# * `$<` is set to a file containing the image ID
#
# Additionally, you can override the EXPR on a per-image basis
# by overriding the `docker.tag.GROUP` variable on a per-target
# basis:
#
# IMAGE.docker.tag.GROUP: docker.tag.GROUP = EXPR
#
# > For example:
# >
# > For the mast part, the Ambassador Pro images are
# > - built as : `docker/$(NAME).docker`
# > - built from: `docker/$(NAME)/Dockerfile`
# > - pushed as : `quay.io/datawire/ambassador_pro:$(NAME)-$(VERSION)`
# > However, as an exception, the Ambassador Core image is
# > - built as : `ambassador/ambassador.docker`
# > - pushed as : `quay.io/datawire/ambassador_pro:amb-core-$(VERSION)`
# >
# > Additionally, we want to be able to push to a private
# > in-cluster registry for testing before we do a release. The
# > tag names pushed to the cluster should be based on the image
# > ID, so that we don't need to configure a funny
# > ImagePullPolicy during testing.
# >
# > We accomplish this by saying:
# >
# > docker.tag.release = quay.io/datawire/ambassador_pro:$(notdir $*)-$(VERSION)
# > include build-aux/docker-cluster.mk # docker-cluster.mk sets the `docker.tag.cluster` variable
# > include build-aux/docker.mk
# > # The above will cause docker.mk to define targets:
# > # - %.docker.tag.release
# > # - %.docker.push.release
# > # - %.docker.tag.cluster
# > # - %.docker.push.cluster
# >
# > # Override the release name a specific image.
# > # Release ambassador/ambassador.docker
# > # - based on the above : quay.io/datawire/ambassador_pro:ambassador-$(VERSION)
# > # - after being overridden: quay.io/datawire/ambassador_pro:amb-core-$(VERSION)
# > ambassador/ambassador.docker.tag.release: docker.tag.release = quay.io/datawire/ambassador_pro:amb-core-$(VERSION)
# >
# > and having our
# > - `build` target depend on `NAME.docker.tag.release` (for each NAME).
#
# ## Pushing ##
#
# Pushing a tag: You can push tags that have been created with
# `IMAGE.docker.tag.GROUP` (see above) by depending on
# `IMAGE.docker.push.GROUP`.
#
# > For example:
# >
# > Based on the above Ambassador Pro example in the "Tagging"
# > section, we have our
# > - `check` target depend on `NAME.docker.push.cluster` (for each NAME).
# > - `release` target depend on `NAME.docker.push.release` (for each NAME).
#
# ## Cleaning ##
#
# - Clean up: You can untag (if there are any tags) and remove an
# image by having your `clean` target depend on
Expand All @@ -113,6 +143,63 @@
# NOT automatically added to the common.mk 'clean' target, and
# you MUST do that yourself.
#
# ## More build-rule examples ##
#
# 1. You might want to specify `docker build` arguments, like
# `--build-arg=` or `-f`:
#
# # Set a custom --build-arg, and use a funny Dockerfile name
# myimage.docker: $(MOVE_IFCHANGED) FORCE
# docker build --iidfile=$(@D)/.tmp.$(@F).tmp --build-arg=FOO=BAR -f Dockerfile.myimage .
# $(MOVE_IFCHANGED) $(@D)/.tmp.$(@F).tmp $@
#
# 2. In `ambassador.git`, building the Envoy binary is slow, so we
# might want to try pulling it from a build-cache Docker
# repository, instead of building it locally:
#
# # Building this is expensive, so try grabbing a cached
# # version before trying to build it. This goes ahead and
# # tags the image, for caching purposes.
# base-envoy.docker: $(WRITE_IFCHANGED) $(var.)BASE_ENVOY_IMAGE_CACHE
# if ! docker run --rm --entrypoint=true $(BASE_ENVOY_IMAGE_CACHE); then \
# $(MAKE) envoy-bin/envoy-static-stripped
# docker build -t $(BASE_ENVOY_IMAGE_CACHE) -f Dockerfile.base-envoy; \
# fi
# docker image inspect $(BASE_ENVOY_IMAGE_CACHE) --format='{{.Id}}' | $(WRITE_IFCHANGED) $@
#
# 3. In `apro.git`, we have many Docker images to build; each with
# a Dockerfile at `docker/NAME/Dockerfile`. We accomplish this
# with a simple pattern rule only slightly more complex than
# the one given in the "Building" section:
#
# %.docker: %/Dockerfile $(MOVE_IFCHANGED) FORCE
# # Try with --pull, fall back to without --pull
# docker build --iidfile=$(@D)/.tmp.$(@F).tmp --pull $* || docker build --iidfile=$(@D)/.tmp.$(@F).tmp $*
# $(MOVE_IFCHANGED) $(@D)/.tmp.$(@F).tmp $@
#
# The `--pull` is a good way to ensure that we incorporate any
# patches to the base images that we build ours FROM. However,
# sometimes `--pull` doesn't work, because in at least one case
# the the Dockerfile refers to a local image ID hash from a
# previously built image; trying to pull that ID hash will
# fail.
#
# For many of these images, we have have Makefile-built
# artifacts that we would like to include in the image. We
# accomplish this by simply declaring dependencies off of the
# `docker/NAME.docker` files, and writing rules to copy the
# artifacts in to the `docker/NAME/` directory:
#
# # In this example, the `docker/app-sidecar/Dockerfile` image
# # needs an already-compiled `ambex` binary.
#
# # Declare the dependency...
# docker/app-sidecar.docker: docker/app-sidecar/ambex
#
# # ... and copy it in to the Docker context
# docker/app-sidecar/ambex: bin_linux_amd64/ambex
# cp $< $@

ifeq ($(words $(filter $(abspath $(lastword $(MAKEFILE_LIST))),$(abspath $(MAKEFILE_LIST)))),1)
_docker.mk := $(lastword $(MAKEFILE_LIST))
include $(dir $(_docker.mk))prelude.mk
Expand All @@ -138,13 +225,6 @@ docker.LOCALHOST = $(if $(filter darwin,$(GOHOSTOS)),host.docker.internal,localh
#
# Output targets

# file contents:
# line 1: image ID
%.docker: %/Dockerfile $(MOVE_IFCHANGED) FORCE
# Try with --pull, fall back to without --pull
docker build --iidfile=$(@D)/.tmp.$(@F).tmp --pull $* || docker build --iidfile=$(@D)/.tmp.$(@F).tmp $*
$(MOVE_IFCHANGED) $(@D)/.tmp.$(@F).tmp $@

%.docker.clean: $(addprefix %.docker.clean.,$(_docker.clean.groups))
if [ -e $*.docker ]; then docker image rm "$$(cat $*.docker)" || true; fi
rm -f $*.docker
Expand Down
2 changes: 1 addition & 1 deletion build-aux/docs/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ you ignore that exit code:

## Adding dependencies of tests

It is reasonassumed that *all* tests depend on `make build`. To add a
It is assumed that *all* tests depend on `make build`. To add a
dependency shared by all tests, to declare a dependency that all tests
should depend on, declare it as a dependency of `check` itself. For
example, `common.mk` says:
Expand Down
2 changes: 1 addition & 1 deletion build-aux/k8s.mk
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ _k8s.push = $(addsuffix .docker.push.cluster,$(K8S_IMAGES))
apply: $(filter-out $(wildcard $(_k8s.push)),$(_k8s.push))
deploy: $(_k8s.push)
apply deploy: $(KUBECONFIG) $(KUBEAPPLY) $(K8S_ENVS)
$(if $(K8S_ENVS),set -a && $(foreach k8s_env,$(abspath $(K8S_ENVS)), . $(k8s_env) && ))$(KUBEAPPLY) -t 300 $(addprefix -f ,$(K8S_DIRS))
$(if $(K8S_ENVS),set -a && $(foreach k8s_env,$(abspath $(K8S_ENVS)), . $(k8s_env) && ))$(KUBEAPPLY) --timeout=5m $(addprefix -f ,$(K8S_DIRS))
.PHONY: apply deploy

$(KUBECONFIG).clean: _clean-k8s
Expand Down

0 comments on commit 2470958

Please sign in to comment.