From 8a53080f051ffab2b52c7a740d1dadb7834df4a7 Mon Sep 17 00:00:00 2001 From: cong Date: Fri, 5 Jan 2024 16:53:03 +0800 Subject: [PATCH] try github ci --- .github/VERSION | 1 + .../crd.alauda.io_alaudaloadbalancer2.yaml | 355 ++++++++++++++++++ .github/chart/alb/values.yaml | 34 ++ .github/cr.yaml | 1 + .github/workflows/build.yaml | 121 ++++++ .github/workflows/test.yaml | 0 Dockerfile | 7 +- docs/README.md | 122 +++++- scripts/alb-dev-actions.sh | 1 + scripts/alb-github-actions.sh | 34 ++ template/Dockerfile | 8 +- 11 files changed, 677 insertions(+), 7 deletions(-) create mode 100644 .github/VERSION create mode 100644 .github/chart/alb/crds/crd.alauda.io_alaudaloadbalancer2.yaml create mode 100644 .github/chart/alb/values.yaml create mode 100644 .github/cr.yaml create mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/test.yaml create mode 100644 scripts/alb-github-actions.sh diff --git a/.github/VERSION b/.github/VERSION new file mode 100644 index 00000000..60453e69 --- /dev/null +++ b/.github/VERSION @@ -0,0 +1 @@ +v1.0.0 \ No newline at end of file diff --git a/.github/chart/alb/crds/crd.alauda.io_alaudaloadbalancer2.yaml b/.github/chart/alb/crds/crd.alauda.io_alaudaloadbalancer2.yaml new file mode 100644 index 00000000..e90b3f75 --- /dev/null +++ b/.github/chart/alb/crds/crd.alauda.io_alaudaloadbalancer2.yaml @@ -0,0 +1,355 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.1 + name: alaudaloadbalancer2.crd.alauda.io +spec: + group: crd.alauda.io + names: + kind: ALB2 + listKind: ALB2List + plural: alaudaloadbalancer2 + shortNames: + - alb2 + singular: alaudaloadbalancer2 + scope: Namespaced + versions: + - deprecated: true + deprecationWarning: alb2.v1.crd is deprecated,use alb2.v2beta1.crd instead. + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + address: + description: address is only used to display at front-end. + type: string + bind_address: + description: bind_address is deprecated, default "" + type: string + domains: + description: domains is deprecated, default [] + items: + type: string + type: array + iaas_id: + description: iaas_id is deprecated, default "" + type: string + type: + description: type defines the loadbalance alb2 uses, now only support + nginx + enum: + - nginx + type: string + type: object + status: + properties: + probeTime: + format: int64 + type: integer + reason: + description: reason defines the possible cause of alb2 state change + type: string + state: + description: state defines the status of alb2, the possible values + are ready/warning state:ready means ok state:warning can be caused + by port conflict in alb2 + type: string + type: object + type: object + storage: false + served: false + subresources: + status: {} + - name: v2beta1 + schema: + openAPIV3Schema: + description: ALB2 is the Schema for the alaudaloadbalancer2 API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ALB2Spec defines the desired state of ALB2 + properties: + address: + description: custom address of this alb + type: string + config: + properties: + antiAffinityKey: + type: string + backlog: + type: integer + bindNIC: + type: string + cleanMetricsInterval: + type: integer + defaultIngressClass: + type: boolean + defaultSSLCert: + type: string + defaultSSLStrategy: + type: string + enableALb: + type: boolean + enableCrossClusters: + type: string + enableGC: + type: string + enableGCAppRule: + type: string + enableGoMonitor: + type: boolean + enableGzip: + type: string + enableHTTP2: + type: string + enableIPV6: + type: string + enableIngress: + type: string + enablePortProject: + type: boolean + enablePortprobe: + type: string + enableProfile: + type: boolean + enablePrometheus: + type: string + gateway: + properties: + enable: + type: boolean + mode: + type: string + name: + type: string + type: object + goMonitorPort: + type: integer + ingressController: + type: string + ingressHTTPPort: + type: integer + ingressHTTPSPort: + type: integer + loadbalancerName: + type: string + loadbalancerType: + type: string + maxTermSeconds: + type: integer + metricsPort: + type: integer + networkMode: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + overwrite: + properties: + configmap: + items: + properties: + name: + type: string + target: + type: string + type: object + type: array + image: + items: + properties: + alb: + type: string + nginx: + type: string + target: + type: string + type: object + type: array + type: object + policyZip: + type: boolean + portProjects: + type: string + projects: + items: + type: string + type: array + reloadtimeout: + type: integer + replicas: + type: integer + resources: + properties: + alb: + properties: + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + type: object + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + type: object + resyncPeriod: + type: integer + syncPolicyInterval: + type: integer + vip: + properties: + allocateLoadBalancerNodePorts: + type: boolean + enableLbSvc: + type: boolean + lbSvcAnnotations: + additionalProperties: + type: string + type: object + lbSvcIpFamilyPolicy: + description: IPFamilyPolicy represents the dual-stack-ness + requested or required by a Service + type: string + type: object + workerLimit: + type: integer + type: object + x-kubernetes-preserve-unknown-fields: true + type: + enum: + - nginx + type: string + type: object + status: + description: ALB2Status defines the observed state of ALB2, detail in + ALB2StatusDetail + properties: + detail: + properties: + address: + description: status set by operator + properties: + host: + items: + type: string + type: array + ipv4: + items: + type: string + type: array + ipv6: + items: + type: string + type: array + msg: + type: string + ok: + type: boolean + type: object + alb: + description: status set by alb itself + properties: + portstatus: + additionalProperties: + properties: + conflict: + type: boolean + msg: + type: string + probeTimeStr: + format: date-time + type: string + type: object + description: port status of this alb. key format protocol-port + type: object + type: object + deploy: + description: status set by operator + properties: + probeTimeStr: + format: date-time + type: string + reason: + type: string + state: + type: string + type: object + version: + description: status set by operator + properties: + imagePatch: + type: string + version: + type: string + type: object + type: object + probeTime: + format: int64 + type: integer + probeTimeStr: + format: date-time + type: string + reason: + description: reason defines the possible cause of alb2 state change + type: string + state: + default: Pending + description: state defines the status of alb2, the possible values + are ready/warning state:ready means ok state:warning can be caused + by port conflict in alb2 + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/.github/chart/alb/values.yaml b/.github/chart/alb/values.yaml new file mode 100644 index 00000000..da01ce45 --- /dev/null +++ b/.github/chart/alb/values.yaml @@ -0,0 +1,34 @@ +## global +operator: + albImagePullPolicy: IfNotPresent +defaultAlb: false # 部署默认的alb +operatorReplicas: 1 +operatorDeployMode: "deployment" # csv|deployment +global: + labelBaseDomain: cpaas.io + namespace: kube-system + registry: + address: theseedoaa + images: + alb2: + code: https://github.com/alauda/alb + support_arm: true + repository: alb + nginx: + code: https://github.com/alauda/alb + support_arm: true + repository: alb-nginx +resources: + alb: + limits: + cpu: "200m" + memory: 2Gi + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 50m + memory: 128Mi diff --git a/.github/cr.yaml b/.github/cr.yaml new file mode 100644 index 00000000..4e1180df --- /dev/null +++ b/.github/cr.yaml @@ -0,0 +1 @@ +sign: false \ No newline at end of file diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000..9215230c --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,121 @@ +name: Build + +on: + pull_request: + branches: + - master + - release-* + paths-ignore: + - 'docs/**' + - '**.md' + - '.github/VERSION' + push: + branches: + - master + - release-* + paths-ignore: + - 'docs/**' + - '**.md' + - '.github/VERSION' + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +env: + GO_VERSION: '' + GOSEC_VERSION: '2.18.2' + HELM_VERSION: v3.13.3 + SUBMARINER_VERSION: '0.16.2' + IMAGE_REPO: "theseedoaa" +jobs: + build-alb: + name: Build alb + runs-on: ubuntu-22.04 + steps: + - uses: jlumbroso/free-disk-space@v1.3.1 + with: + android: true + dotnet: true + haskell: true + docker-images: false + large-packages: false + tool-cache: false + swap-storage: false + + - uses: actions/checkout@v4 + - name: version + uses: HardNorth/github-version-generate@v1.3.0 + with: + version-source: file + version-file: .github/VERSION + version-file-extraction-pattern: 'v(.+)' + - uses: docker/setup-buildx-action@v3 + - uses: azure/setup-helm@v3 + with: + version: '${{ env.HELM_VERSION }}' + - name: build + run: | + set -x + source ./scripts/alb-dev-actions.sh + export VERSION=$(alb-github-gen-version) + + OPENRESTY_BASE=theseedoaa/alb-nginx-base:v1.22.0 + GO_BUILD_BASE=golang:1.18.10-alpine3.16 + RUN_BASE=theseedoaa/ops-alpine:3.17 # we need nonroot user + + # build images + docker buildx build --platform linux/amd64 -t $IMAGE_REPO/alb:$VERSION --build-arg VERSION=$VERSION --build-arg RUN_BASE=$RUN_BASE --build-arg BUILD_BASE=$GO_BUILD_BASE -o type=docker -f ./Dockerfile . + + docker pull $OPENRESTY_BASE + docker buildx build --platform linux/amd64 -t $IMAGE_REPO/alb-nginx:$VERSION --build-arg VERSION=$VERSION --build-arg OPENRESTY_BASE=$OPENRESTY_BASE --build-arg BUILD_BASE=$GO_BUILD_BASE -o type=docker -f ./template/Dockerfile ./ + + docker images + + docker save $IMAGE_REPO/alb:$VERSION > alb.tar + docker save $IMAGE_REPO/alb-nginx:$VERSION > alb-nginx.tar + ls + env + # build chart + alb-build-github-chart $IMAGE_REPO $VERSION ./deploy/chart/alb ./alauda-alb2.tgz + - name: Upload alb images to artifact + uses: actions/upload-artifact@v3 + with: + name: alb + path: alb.tar + - name: Upload alb-nginx images to artifact + uses: actions/upload-artifact@v3 + with: + name: alb-nginx + path: alb-nginx.tar + - name: Upload chart's to artifact + uses: actions/upload-artifact@v3 + with: + name: alb-chart + path: alauda-alb2.tgz + - name: sync docker + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_CLI_EXPERIMENTAL: enabled + # if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + run: | + source ./scripts/alb-dev-actions.sh + export VERSION=$(alb-github-gen-version) + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + docker push $IMAGE_REPO/alb:$VERSION + docker push $IMAGE_REPO/alb-nginx:$VERSION + - name: sync chart + # if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + uses: helm/chart-releaser-action@v1.6.0 + with: + charts_dir: ./deploy/chart/alb + config: .github/cr.yaml + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + - name: bump version + # if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + run: | + source ./scripts/alb-dev-actions.sh + git status + git diff \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000..e69de29b diff --git a/Dockerfile b/Dockerfile index cdd9aecf..455a5c39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,7 @@ -FROM build-harbor.alauda.cn/ops/golang:1.18-alpine3.16 AS builder +ARG BUILD_BASE=build-harbor.alauda.cn/ops/golang:1.18-alpine3.16 +ARG RUN_BASE=build-harbor.alauda.cn/ops/alpine:3.17 + +FROM ${BUILD_BASE} AS builder ENV GO111MODULE=on ENV GOPROXY=https://goproxy.cn,direct @@ -11,7 +14,7 @@ RUN go build -buildmode=pie -ldflags '-w -s -linkmode=external -extldflags=-Wl,- RUN go build -buildmode=pie -ldflags '-w -s -linkmode=external -extldflags=-Wl,-z,relro,-z,now' -v -o /out/migrate/init-port-info alauda.io/alb2/migrate/init-port-info RUN go build -buildmode=pie -ldflags '-w -s -linkmode=external -extldflags=-Wl,-z,relro,-z,now' -v -o /out/operator alauda.io/alb2/cmd/operator -FROM build-harbor.alauda.cn/ops/alpine:3.17 AS base +FROM $RUN_BASE AS base RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories RUN apk update && apk add --no-cache iproute2 jq libcap && rm -rf /usr/bin/nc diff --git a/docs/README.md b/docs/README.md index 6688f945..ee138cba 100644 --- a/docs/README.md +++ b/docs/README.md @@ -88,8 +88,124 @@ EOF now you could `curl http://${ip}` # more advance config ## ft and rule +除了ingress之外，可以通过创建ft和rule，来指定alb的路由规则 +```yaml +apiVersion: crd.alauda.io/v1 +kind: Frontend +metadata: + labels: + alb2.cpaas.io/name: alb-demo # 必须要通过label指定这个ft属于那个alb + name: alb-demo-00080 + namespace: kube-system +spec: + backendProtocol: "" # 转发到的后端的协议 + certificate_name: "" # 如果是https，希望配置默认证书，通过这个字段设置。格式为 + port: 80 # 这个端口本身的协议 + protocol: http +--- +apiVersion: crd.alauda.io/v1 +kind: Rule +metadata: + labels: + alb2.cpaas.io/frontend: alb-demo-00080 # 必须要通过label指定这个rule属于那个ft + alb2.cpaas.io/name: alb-demo # 必须要通过label指定这个rule属于那个alb + name: alb-demo-00080-topu + namespace: kube-system +spec: + backendProtocol: "" + certificate_name: "" + corsAllowHeaders: "" + corsAllowOrigin: "" + description: default/hello-world:80:0 + dsl: '[{[[STARTS_WITH /]] URL }]' # stringilze的匹配规则，用来被搜索 + dslx: # 匹配规则，详细配置见 TODO 匹置规则 + - type: URL + values: + - - STARTS_WITH + - / + enableCORS: false + priority: 5 # 规则优先级，越小越会优先匹配 + redirectCode: 0 + redirectURL: "" + rewrite_base: / + serviceGroup: + services: + - name: hello-world + namespace: default + port: 80 + weight: 100 + url: / +``` ## ingress ### ingress with other port -### support annotation -## container mode -## gatewayapi \ No newline at end of file +### supported annotation +#### alb only +##### rewrite request +headers_add 只会在没有这个header是才添加 +alb.ingress.cpaas.io/rewrite-request +```yaml +alb.ingress.cpaas.io/rewrite-request: | +{"headers_remove":["h1"],"headers":{"a":"b"},"headers_add":{"aa","bb"}} +``` +##### rewrite response +alb.ingress.cpaas.io/rewrite-response +```yaml +alb.ingress.cpaas.io/rewrite-response: | +{"headers_remove":["h1"],"headers":{"a":"b"},"headers_add":{"aa","bb"}} +``` + +#### compatiable with ingress-nginx +##### rewrite + nginx.ingress.kubernetes.io/rewrite-target +##### cors + nginx.ingress.kubernetes.io/enable-cors + nginx.ingress.kubernetes.io/cors-allow-headers + nginx.ingress.kubernetes.io/cors-allow-origin +##### backend + nginx.ingress.kubernetes.io/backend-protocol +##### redirect + nginx.ingress.kubernetes.io/temporal-redirect + nginx.ingress.kubernetes.io/permanent-redirect +##### vhost + nginx.ingress.kubernetes.io/upstream-vhost" +## alb 配置 +### 容器网络模式 +alb默认是以主机网络部署的，这样的好处是可以通过节点ip直接访问，缺点是每个alb只能独占节点，或者需要手动管理alb的端口。 +同样alb也支持容器网络模式部署，并通过lbsvc来提供对外访问的能力 +```yaml +apiVersion: crd.alauda.io/v2beta1 +kind: ALB2 +metadata: + name: alb-demo + namespace: kube-system +spec: + type: "nginx" + config: + networkMode: container # 部署的alb pod使用容器网络模式 + vip: + enableLbSvc: true # 自动创建一个loadbalancer类型的svc，并且将分配给svc的地址当作alb的地址 + loadbalancerName: alb-demo + nodeSelector: + alb-demo: "true" + projects: + - ALL_ALL + replicas: 1 +``` +### gatewayapi +alb原生支持gatewayapi，只要创建gateway的时候指定gatewayclass为``即可 +```yaml +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: Gateway +metadata: + name: g1 + namespace: g1 +spec: + gatewayClassName: exclusive-gateway + listeners: + - name: http + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: All +``` \ No newline at end of file diff --git a/scripts/alb-dev-actions.sh b/scripts/alb-dev-actions.sh index 13027484..5e5e3ef9 100755 --- a/scripts/alb-dev-actions.sh +++ b/scripts/alb-dev-actions.sh @@ -4,6 +4,7 @@ source ./scripts/alb-test-actions.sh source ./scripts/alb-lint-actions.sh source ./scripts/alb-build-actions.sh source ./scripts/alb-deploy-actions.sh +source ./scripts/alb-github-actions.sh source ./template/actions/alb-nginx.sh function alb-cleanup() { diff --git a/scripts/alb-github-actions.sh b/scripts/alb-github-actions.sh new file mode 100644 index 00000000..58607723 --- /dev/null +++ b/scripts/alb-github-actions.sh @@ -0,0 +1,34 @@ +#!/bin/bash +function alb-github-gen-version() { + local branch=$(echo "$GITHUB_HEAD_REF" | sed 's|/|-|g') + echo "v$CURRENT_VERSION-$branch.$GITHUB_RUN_NUMBER.$GITHUB_RUN_ATTEMPT" +} + +function alb-github-sync() { + docker load <./alb/alb.tar + docker load <./alb-nginx/alb-nginx.tar + kind-load-image-in-current theseedoaa/alb-nginx:$(yq .global.images.alb2.tag ./alauda-alb2/values.yaml) + kind-load-image-in-current theseedoaa/alb:$(yq .global.images.alb2.tag ./alauda-alb2/values.yaml) +} + +function alb-build-github-chart() { + # alb-build-github-chart $RELEASE_TAG ./chart/alb ./alb-chart.tar + local repo=$1 + local version=$2 + local chart_dir=$3 + local out_path=$4 + + local branch=$GITHUB_HEAD_REF + local commit=$GITHUB_SHA + cp ./.github/chart/alb/values.yaml $chart_dir + cp ./.github/chart/alb/crds/crd.alauda.io_alaudaloadbalancer2.yaml $chart_dir/crds # do not served v1 + yq -i e ".global.images.alb2.tag |= \"$version\"" $chart_dir/values.yaml + yq -i e ".global.registry.address |= \"$repo\"" $chart_dir/values.yaml + yq -i e ".global.images.nginx.tag |= \"$version\"" $chart_dir/values.yaml + yq -i e ".annotations.branch |= \"$branch\"" $chart_dir/Chart.yaml + yq -i e ".annotations.commit |= \"$commit\"" $chart_dir/Chart.yaml + + helm package --debug $chart_dir --version $version + mv ./alauda-alb2-$version.tgz $out_path + return +} diff --git a/template/Dockerfile b/template/Dockerfile index b07fcd6c..69b79d9a 100644 --- a/template/Dockerfile +++ b/template/Dockerfile @@ -1,5 +1,9 @@ + +ARG BUILD_BASE=build-harbor.alauda.cn/ops/golang:1.18-alpine3.16 +ARG OPENRESTY_BASE=build-harbor.alauda.cn/3rdparty/alb-nginx:v1.22.0 + # build nignx/openresty base image -FROM build-harbor.alauda.cn/ops/golang:1.18-alpine3.15 AS tweak_gen +FROM ${BUILD_BASE} AS tweak_gen COPY ./ /alb/ ENV GO111MODULE=on ENV GOPROXY=https://goproxy.cn,direct @@ -10,7 +14,7 @@ RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositorie RUN cd cmd/utils/tweak_gen && go build -buildmode=pie -ldflags '-w -s -linkmode=external -extldflags=-Wl,-z,relro,-z,now' -v -o /alb/out/tweak_gen alauda.io/alb2/cmd/utils/tweak_gen RUN ls -alh /alb/out/ -FROM build-harbor.alauda.cn/3rdparty/alb-nginx:v1.22.0 +FROM ${OPENRESTY_BASE} # keep it as same as pkg/config/albrun.go ENV OLD_CONFIG_PATH /etc/alb2/nginx/nginx.conf