diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 7760e34..461bc82 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -30,7 +30,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-4 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - /bin/bash - -c @@ -60,35 +60,34 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-4 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/upload-gocache.sh" resources: requests: cpu: 100m memory: 1Gi - - # - name: ci-push-kubelb-charts - # always_run: true - # decorate: true - # clone_uri: "ssh://git@github.com/kubermatic/kubelb.git" - # branches: - # # Match on tags - # - ^v\d+\.\d+\.\d+.* - # reporter_config: - # slack: - # channel: dev-kubelb - # labels: - # preset-docker-push: "true" - # preset-goproxy: "true" - # spec: - # containers: - # - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-4 - # command: - # - make - # args: - # - release-charts - # resources: - # requests: - # cpu: 100m - # memory: 500m +# - name: ci-push-kubelb-charts +# always_run: true +# decorate: true +# clone_uri: "ssh://git@github.com/kubermatic/kubelb.git" +# branches: +# # Match on tags +# - ^v\d+\.\d+\.\d+.* +# reporter_config: +# slack: +# channel: dev-kubelb +# labels: +# preset-docker-push: "true" +# preset-goproxy: "true" +# spec: +# containers: +# - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 +# command: +# - make +# args: +# - release-charts +# resources: +# requests: +# cpu: 100m +# memory: 500m diff --git a/.prow/verify.yaml b/.prow/verify.yaml index fb0c1b3..c71bb0c 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -39,7 +39,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golangci/golangci-lint:v1.59.0 + - image: golangci/golangci-lint:v1.59.1 command: - make args: @@ -57,7 +57,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-4 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - make args: @@ -72,7 +72,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/kubelb.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-4 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - make args: @@ -84,7 +84,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/kubelb.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-4 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - make args: @@ -142,7 +142,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-4 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 securityContext: privileged: true env: @@ -169,3 +169,47 @@ presubmits: limits: memory: 16Gi cpu: 4 + + - name: pull-kubelb-verify-shfmt + run_if_changed: "^hack/" + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubelb.git" + spec: + containers: + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + command: + - shfmt + args: + # -l list files whose formatting differs from shfmt's + # -d error with a diff when the formatting differs + # -i uint indent: 0 for tabs (default), >0 for number of spaces + # -sr redirect operators will be followed by a space + - "-l" + - "-sr" + - "-i" + - "2" + - "-d" + - "hack" + resources: + requests: + memory: 32Mi + cpu: 50m + limits: + memory: 256Mi + cpu: 250m + + - name: pull-kubelb-license-validation + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/kubelb.git" + labels: + preset-goproxy: "true" + spec: + containers: + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + command: + - ./hack/verify-licenses.sh + resources: + requests: + memory: 2Gi + cpu: 2 diff --git a/.wwhrd.yml b/.wwhrd.yml new file mode 100644 index 0000000..0907766 --- /dev/null +++ b/.wwhrd.yml @@ -0,0 +1,30 @@ +# Copyright 2024 The KubeLB Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +blacklist: + - GPL-2.0 + +whitelist: + - Apache-2.0 + - BSD-2-Clause + - BSD-3-Clause + - FreeBSD + - ISC + - LGPL-3.0 + - MIT + - MPL-2.0 + - NewBSD + +exceptions: + - github.com/ajeddeloh/go-json # Since it's a fork, https://github.com/golang/go/blob/master/LICENSE diff --git a/Makefile b/Makefile index 8fc62af..b07d0d2 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,13 @@ SHELL = /bin/bash -eu -o pipefail # Image URL to use all building/pushing image targets KUBELB_IMG ?= quay.io/kubermatic/kubelb-manager KUBELB_CCM_IMG ?= quay.io/kubermatic/kubelb-ccm + +## Tool Versions # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.28.0 +ENVTEST_K8S_VERSION = 1.30.0 +KUSTOMIZE_VERSION ?= v5.4.3 +CONTROLLER_TOOLS_VERSION ?= v0.15.0 +GO_VERSION = 1.22.5 export GOPATH?=$(shell go env GOPATH) export CGO_ENABLED=0 @@ -13,10 +18,11 @@ export GO111MODULE=on export GOFLAGS?=-mod=readonly -trimpath export GIT_TAG ?= $(shell git tag --points-at HEAD) -GO_VERSION = 1.22.2 - IMAGE_TAG = \ $(shell echo $$(git rev-parse HEAD && if [[ -n $$(git status --porcelain) ]]; then echo '-dirty'; fi)|tr -d ' ') + +VERSION = $(shell cat VERSION) + CCM_IMAGE_NAME ?= $(KUBELB_CCM_IMG):$(IMAGE_TAG) KUBELB_IMAGE_NAME ?= $(KUBELB_IMG):$(IMAGE_TAG) @@ -71,6 +77,8 @@ manifests: generate controller-gen ## Generate WebhookConfiguration, ClusterRole generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate/boilerplate.go.txt" paths="./..." +update-codegen: generate controller-gen manifests fmt vet go-mod-tidy + .PHONY: fmt fmt: ## Run go fmt against code. go fmt ./... @@ -88,6 +96,9 @@ yamllint: ## Run yamllint against code. check-dependencies: ## Verify go.mod. go mod verify +go-mod-tidy: + go mod tidy + verify-boilerplate: ## Run verify-boilerplate code. ./hack/verify-boilerplate.sh @@ -185,10 +196,6 @@ KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest -## Tool Versions -KUSTOMIZE_VERSION ?= v5.3.0 -CONTROLLER_TOOLS_VERSION ?= v0.14.0 - KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. @@ -205,12 +212,19 @@ envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest +.PHONY: shfmt +shfmt: + shfmt -w -sr -i 2 hack + HELM_DOCS ?= $(LOCALBIN)/helm-docs .PHONY: helm-docs helm-docs: $(HELM_DOCS) ## Download helm-docs locally if necessary. $(HELM_DOCS): $(LOCALBIN) - test -s $(LOCALBIN)/helm-docs || GOBIN=$(LOCALBIN) go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.11.2 + test -s $(LOCALBIN)/helm-docs || GOBIN=$(LOCALBIN) go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.14.2 + +helm-lint: + helm lint charts/* generate-helm-docs: helm-docs $(LOCALBIN)/helm-docs charts/ @@ -222,5 +236,5 @@ bump-chart: $(SED) -i "s/tag:.*/tag: $(IMAGE_TAG)/" charts/*/values.yaml .PHONY: release-charts helm-docs generate-helm-docs -release-charts: bump-chart +release-charts: helm-lint generate-helm-docs bump-chart CHART_VERSION=$(IMAGE_TAG) ./hack/release-helm-charts.sh \ No newline at end of file diff --git a/api/kubelb.k8c.io/v1alpha1/config_types.go b/api/kubelb.k8c.io/v1alpha1/config_types.go index 28ddd7e..dc5aa5e 100644 --- a/api/kubelb.k8c.io/v1alpha1/config_types.go +++ b/api/kubelb.k8c.io/v1alpha1/config_types.go @@ -56,10 +56,12 @@ type ConfigSpec struct { // EnvoyProxy defines the desired state of the EnvoyProxy type EnvoyProxy struct { - // Topology defines the deployment topology for Envoy Proxy. Valid values are: shared, dedicated, and global. // +kubebuilder:validation:Enum=shared;dedicated;global // +kubebuilder:default=shared - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + // +kubebuilder:validation:XValidation:rule="self == oldSelf || (self != oldSelf && oldSelf == 'dedicated')",message="Value is immutable and only allowed change is from dedicated(deprecated) to shared/global" + + // Topology defines the deployment topology for Envoy Proxy. Valid values are: shared and global. + // DEPRECATION NOTICE: The value "dedicated" is deprecated and will be removed in a future release. Dedicated topology will now default to shared topology. // +optional Topology EnvoyProxyTopology `json:"topology,omitempty"` @@ -70,7 +72,7 @@ type EnvoyProxy struct { // Replicas defines the number of replicas for Envoy Proxy. This field is ignored if UseDaemonset is set to true. // +kubebuilder:validation:Minimum=1 - // +kubeblider:default=3 + // +kubebuilder:default=3 // +optional Replicas int32 `json:"replicas,omitempty"` diff --git a/api/kubelb.k8c.io/v1alpha1/loadbalancer_types.go b/api/kubelb.k8c.io/v1alpha1/loadbalancer_types.go index 1536af5..515f021 100644 --- a/api/kubelb.k8c.io/v1alpha1/loadbalancer_types.go +++ b/api/kubelb.k8c.io/v1alpha1/loadbalancer_types.go @@ -102,6 +102,8 @@ type LoadBalancerSpec struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:shortName=lb +// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb.k8c.io/origin-name",name="OriginName",type="string" +// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb.k8c.io/origin-ns",name="OriginNamespace",type="string" // +genclient // LoadBalancer is the Schema for the loadbalancers API diff --git a/api/kubelb.k8c.io/v1alpha1/route_types.go b/api/kubelb.k8c.io/v1alpha1/route_types.go index a0d608a..1c18c7e 100644 --- a/api/kubelb.k8c.io/v1alpha1/route_types.go +++ b/api/kubelb.k8c.io/v1alpha1/route_types.go @@ -92,6 +92,9 @@ type UpstreamReferenceGrant struct { //+kubebuilder:object:root=true //+kubebuilder:subresource:status +// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb.k8c.io/origin-name",name="OriginName",type="string" +// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb.k8c.io/origin-ns",name="OriginNamespace",type="string" +// +kubebuilder:printcolumn:JSONPath=".metadata.labels.kubelb.k8c.io/origin-resource-kind",name="OriginResource",type="string" // Route is the object that represents a route in the cluster. type Route struct { diff --git a/ccm.dockerfile b/ccm.dockerfile index 597a8e3..173067f 100644 --- a/ccm.dockerfile +++ b/ccm.dockerfile @@ -12,20 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM docker.io/golang:1.22.2 as builder +FROM docker.io/golang:1.22.5 as builder -WORKDIR /go/src/k8c.io/kubelb -COPY . . -RUN make build-ccm +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download -FROM gcr.io/distroless/static:nonroot - -WORKDIR / +# Copy the go source +COPY cmd/ cmd/ +COPY api/ api/ +COPY internal/ internal/ -COPY --from=builder \ - /go/src/k8c.io/kubelb/bin/ccm \ - /usr/local/bin/ +RUN CGO_ENABLED=0 go build -a -o ccm cmd/ccm/main.go +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/ccm . USER 65532:65532 -ENTRYPOINT ["/usr/local/bin/ccm"] +ENTRYPOINT ["/ccm"] diff --git a/charts/kubelb-ccm/Chart.yaml b/charts/kubelb-ccm/Chart.yaml index 348ca4d..f29e72e 100644 --- a/charts/kubelb-ccm/Chart.yaml +++ b/charts/kubelb-ccm/Chart.yaml @@ -1,6 +1,7 @@ apiVersion: v2 name: kubelb-ccm description: Helm chart for KubeLB CCM +icon: https://raw.githubusercontent.com/kubermatic/kubelb/main/docs/kubelb-logo.png type: application maintainers: - name: Kubermatic diff --git a/charts/kubelb-ccm/README.md b/charts/kubelb-ccm/README.md index 4cf921f..68a68be 100644 --- a/charts/kubelb-ccm/README.md +++ b/charts/kubelb-ccm/README.md @@ -48,9 +48,16 @@ helm install kubelb-ccm kubelb-ccm --namespace kubelb -f values.yaml | image.tag | string | `"v1.0.0"` | | | imagePullSecrets | list | `[]` | | | kubelb.clusterSecretName | string | `"kubelb-cluster"` | | +| kubelb.disableGRPCRouteController | bool | `false` | | +| kubelb.disableGatewayController | bool | `false` | disableGatewayController specifies whether to disable the Gateway Controller. | +| kubelb.disableHTTPRouteController | bool | `false` | | +| kubelb.disableIngressController | bool | `false` | disableIngressController specifies whether to disable the Ingress Controller. | | kubelb.enableLeaderElection | bool | `true` | | | kubelb.nodeAddressType | string | `"InternalIP"` | | | kubelb.tenantName | string | `nil` | | +| kubelb.useGatewayClass | bool | `true` | useGatewayClass specifies whether to target resources with `kubelb` gateway class or all resources. | +| kubelb.useIngressClass | bool | `true` | useIngressClass specifies whether to target resources with `kubelb` ingress class or all resources. | +| kubelb.useLoadBalancerClass | bool | `false` | useLoadBalancerClass specifies whether to target services of type LoadBalancer with `kubelb` load balancer class or all services of type LoadBalancer. | | nameOverride | string | `""` | | | nodeSelector | object | `{}` | | | podAnnotations | object | `{}` | | diff --git a/charts/kubelb-ccm/templates/NOTES.txt b/charts/kubelb-ccm/templates/NOTES.txt new file mode 100644 index 0000000..a473b14 --- /dev/null +++ b/charts/kubelb-ccm/templates/NOTES.txt @@ -0,0 +1,5 @@ +Thank you for installing KubeLB CCM! 🎉 + +Your release is named: {{ .Release.Name }} and exists in namespace: {{ .Release.Namespace }}. + +For more details, please check the official documentation at https://docs.kubermatic.com/kubelb \ No newline at end of file diff --git a/charts/kubelb-ccm/templates/clusterrole.yaml b/charts/kubelb-ccm/templates/clusterrole.yaml index aa7e57e..a8c53fc 100644 --- a/charts/kubelb-ccm/templates/clusterrole.yaml +++ b/charts/kubelb-ccm/templates/clusterrole.yaml @@ -19,6 +19,7 @@ rules: resources: - services verbs: + - create - get - list - patch @@ -32,6 +33,46 @@ rules: - get - patch - update +{{- if not .Values.kubelb.disableGatewayController }} +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways + verbs: + - get + - list + - watch +{{- end }} +{{- if not .Values.kubelb.disableGRPCRouteController }} +- apiGroups: + - gateway.networking.k8s.io + resources: + - grpcroutes + verbs: + - get + - list + - watch +{{- end }} +{{- if not .Values.kubelb.disableHTTPRouteController }} +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io +{{- end }} +{{- if not .Values.kubelb.disableIngressController }} + resources: + - ingresses + verbs: + - get + - list + - watch +{{- end }} --- {{- if .Values.rbac.allowProxyRole }} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kubelb-ccm/templates/deployment.yaml b/charts/kubelb-ccm/templates/deployment.yaml index 2935993..857ad57 100644 --- a/charts/kubelb-ccm/templates/deployment.yaml +++ b/charts/kubelb-ccm/templates/deployment.yaml @@ -36,7 +36,7 @@ spec: - --upstream=http://127.0.0.1:8080/ - --logtostderr=true - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 imagePullPolicy: {{ .Values.image.pullPolicy }} name: kube-rbac-proxy ports: @@ -51,6 +51,28 @@ spec: args: - --enable-leader-election={{ .Values.kubelb.enableLeaderElection }} - --node-address-type={{ default "InternalIP" .Values.kubelb.nodeAddressType }} + - --use-loadbalancer-class={{ default false .Values.kubelb.useLoadBalancerClass }} + {{ if not .Values.kubelb.useGatewayClass -}} + - --use-gateway-class=false + {{ end -}} + {{ if not .Values.kubelb.useIngressClass -}} + - --use-ingress-class=false + {{ end -}} + {{ if .Values.kubelb.disableIngressController -}} + - --disable-ingress-controller=true + {{ end -}} + {{ if .Values.kubelb.disableGatewayController -}} + - --disable-gateway-controller=true + {{ end -}} + {{ if .Values.kubelb.disableHTTPRouteController -}} + - --disable-httproute-controller=true + {{ end -}} + {{ if .Values.kubelb.disableGRPCRouteController -}} + - --disable-grpcroute-controller=true + {{ end -}} + {{ if .Values.kubelb.disableGatewayAPI -}} + - --disable-gateway-api=true + {{ end -}} - --cluster-name={{ required "A valid .Values.kubelb.tenantName to specify the tenant name is required!" .Values.kubelb.tenantName }} env: - name: NAMESPACE diff --git a/charts/kubelb-ccm/values.yaml b/charts/kubelb-ccm/values.yaml index a9feb60..626bb13 100644 --- a/charts/kubelb-ccm/values.yaml +++ b/charts/kubelb-ccm/values.yaml @@ -7,10 +7,28 @@ image: imagePullSecrets: [] kubelb: + ## -- Name of the tenant, must be unique against a load balancer cluster. tenantName: null enableLeaderElection: true nodeAddressType: InternalIP + ## -- Name of the secret that contains kubeconfig for the loadbalancer cluster clusterSecretName: kubelb-cluster + # -- useIngressClass specifies whether to target resources with `kubelb` ingress class or all resources. + useIngressClass: true + # -- useGatewayClass specifies whether to target resources with `kubelb` gateway class or all resources. + useGatewayClass: true + # -- useLoadBalancerClass specifies whether to target services of type LoadBalancer with `kubelb` load balancer class or all services of type LoadBalancer. + useLoadBalancerClass: false + # -- disableGatewayAPI specifies whether to disable the Gateway API and Gateway Controllers. + disableGatewayAPI: false + # -- disableIngressController specifies whether to disable the Ingress Controller. + disableIngressController: false + # -- disableGatewayController specifies whether to disable the Gateway Controller. + disableGatewayController: false + ## -- disableHTTPRouteController specifies whether to disable the HTTPRoute Controller. + disableHTTPRouteController: false + ## -- disableGRPCRouteController specifies whether to disable the GRPCRoute Controller. + disableGRPCRouteController: false extraVolumes: [] extraVolumeMounts: [] diff --git a/charts/kubelb-manager/Chart.yaml b/charts/kubelb-manager/Chart.yaml index b9b4f73..66c9b2e 100644 --- a/charts/kubelb-manager/Chart.yaml +++ b/charts/kubelb-manager/Chart.yaml @@ -1,6 +1,7 @@ apiVersion: v2 name: kubelb-manager description: Helm chart for KubeLB Manager +icon: https://raw.githubusercontent.com/kubermatic/kubelb/main/docs/kubelb-logo.png type: application maintainers: - name: Kubermatic diff --git a/charts/kubelb-manager/README.md b/charts/kubelb-manager/README.md index 5857159..6436249 100644 --- a/charts/kubelb-manager/README.md +++ b/charts/kubelb-manager/README.md @@ -43,7 +43,7 @@ helm install kubelb-manager kubelb-manager --namespace kubelb -f values.yaml | kubelb.envoyProxy.resources | object | `{}` | | | kubelb.envoyProxy.singlePodPerNode | bool | `true` | Deploy single pod per node. | | kubelb.envoyProxy.tolerations | list | `[]` | | -| kubelb.envoyProxy.topology | string | `"shared"` | Topology defines the deployment topology for Envoy Proxy. Valid values are: shared, dedicated, and global. | +| kubelb.envoyProxy.topology | string | `"shared"` | Topology defines the deployment topology for Envoy Proxy. Valid values are: shared and global. | | kubelb.envoyProxy.useDaemonset | bool | `false` | Use DaemonSet for Envoy Proxy deployment instead of Deployment. | | kubelb.propagateAllAnnotations | bool | `false` | Propagate all annotations from the LB resource to the LB service. | | kubelb.propagatedAnnotations | object | `{}` | Allowed annotations that will be propagated from the LB resource to the LB service. | diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml index 7d36922..da7fb93 100644 --- a/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml +++ b/charts/kubelb-manager/crds/kubelb.k8c.io_configs.yaml @@ -978,6 +978,7 @@ spec: Proxy. If specified, the node must have all the indicated labels. type: object replicas: + default: 3 description: Replicas defines the number of replicas for Envoy Proxy. This field is ignored if UseDaemonset is set to true. format: int32 @@ -1086,16 +1087,18 @@ spec: type: array topology: default: shared - description: 'Topology defines the deployment topology for Envoy - Proxy. Valid values are: shared, dedicated, and global.' + description: |- + Topology defines the deployment topology for Envoy Proxy. Valid values are: shared and global. + DEPRECATION NOTICE: The value "dedicated" is deprecated and will be removed in a future release. Dedicated topology will now default to shared topology. enum: - shared - dedicated - global type: string x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf + - message: Value is immutable and only allowed change is from + dedicated(deprecated) to shared/global + rule: self == oldSelf || (self != oldSelf && oldSelf == 'dedicated') useDaemonset: description: |- UseDaemonset defines whether Envoy Proxy will run as daemonset. By default, Envoy Proxy will run as deployment. diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_loadbalancers.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_loadbalancers.yaml index 34f6da8..85d50cb 100644 --- a/charts/kubelb-manager/crds/kubelb.k8c.io_loadbalancers.yaml +++ b/charts/kubelb-manager/crds/kubelb.k8c.io_loadbalancers.yaml @@ -16,7 +16,14 @@ spec: singular: loadbalancer scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .metadata.labels.kubelb.k8c.io/origin-name + name: OriginName + type: string + - jsonPath: .metadata.labels.kubelb.k8c.io/origin-ns + name: OriginNamespace + type: string + name: v1alpha1 schema: openAPIV3Schema: description: LoadBalancer is the Schema for the loadbalancers API diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_routes.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_routes.yaml index 4507d10..9df3edb 100644 --- a/charts/kubelb-manager/crds/kubelb.k8c.io_routes.yaml +++ b/charts/kubelb-manager/crds/kubelb.k8c.io_routes.yaml @@ -14,7 +14,17 @@ spec: singular: route scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .metadata.labels.kubelb.k8c.io/origin-name + name: OriginName + type: string + - jsonPath: .metadata.labels.kubelb.k8c.io/origin-ns + name: OriginNamespace + type: string + - jsonPath: .metadata.labels.kubelb.k8c.io/origin-resource-kind + name: OriginResource + type: string + name: v1alpha1 schema: openAPIV3Schema: description: Route is the object that represents a route in the cluster. diff --git a/charts/kubelb-manager/templates/NOTES.txt b/charts/kubelb-manager/templates/NOTES.txt new file mode 100644 index 0000000..9792299 --- /dev/null +++ b/charts/kubelb-manager/templates/NOTES.txt @@ -0,0 +1,5 @@ +Thank you for installing KubeLB Manager! 🎉 + +Your release is named: {{ .Release.Name }} and exists in namespace: {{ .Release.Namespace }}. + +For more details, please check the official documentation at https://docs.kubermatic.com/kubelb \ No newline at end of file diff --git a/charts/kubelb-manager/templates/clusterrole.yaml b/charts/kubelb-manager/templates/clusterrole.yaml index 2737354..483dcb8 100644 --- a/charts/kubelb-manager/templates/clusterrole.yaml +++ b/charts/kubelb-manager/templates/clusterrole.yaml @@ -11,8 +11,36 @@ rules: resources: - namespaces verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete - get - list + - patch + - update - watch - apiGroups: - "" @@ -50,6 +78,100 @@ rules: - patch - update - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways/status + verbs: + - get + - patch + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - grpcroutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - grpcroutes/status + verbs: + - get + - patch + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/status + verbs: + - get + - patch + - update +- apiGroups: + - kubelb.k8c.io + resources: + - addresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubelb.k8c.io + resources: + - addresses/status + verbs: + - get +- apiGroups: + - kubelb.k8c.io + resources: + - configs + verbs: + - get + - list + - watch +- apiGroups: + - kubelb.k8c.io + resources: + - configs/status + verbs: + - get + - patch + - update - apiGroups: - kubelb.k8c.io resources: @@ -73,19 +195,87 @@ rules: - apiGroups: - kubelb.k8c.io resources: - - configs + - routes verbs: + - create + - delete - get - list + - patch + - update - watch - apiGroups: - kubelb.k8c.io resources: - - configs/status + - routes/status verbs: - get - patch - update +- apiGroups: + - kubelb.k8c.io + resources: + - tenants + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubelb.k8c.io + resources: + - tenants/status + verbs: + - get + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - get + - patch + - update +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch --- {{- if .Values.rbac.allowProxyRole }} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kubelb-manager/templates/deployment.yaml b/charts/kubelb-manager/templates/deployment.yaml index f069d90..535afe7 100644 --- a/charts/kubelb-manager/templates/deployment.yaml +++ b/charts/kubelb-manager/templates/deployment.yaml @@ -36,7 +36,7 @@ spec: - --upstream=http://127.0.0.1:8080/ - --logtostderr=true - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 imagePullPolicy: {{ .Values.image.pullPolicy }} name: kube-rbac-proxy ports: diff --git a/charts/kubelb-manager/values.yaml b/charts/kubelb-manager/values.yaml index b5086a0..0ec1c1a 100644 --- a/charts/kubelb-manager/values.yaml +++ b/charts/kubelb-manager/values.yaml @@ -12,7 +12,7 @@ kubelb: # -- Set to true to skip the generation of the Config CR. Useful when the config CR needs to be managed manually. skipConfigGeneration: false envoyProxy: - # -- Topology defines the deployment topology for Envoy Proxy. Valid values are: shared, dedicated, and global. + # -- Topology defines the deployment topology for Envoy Proxy. Valid values are: shared and global. topology: shared # -- The number of replicas for the Envoy Proxy deployment. replicas: 3 diff --git a/cmd/ccm/main.go b/cmd/ccm/main.go index 19e6e1d..e82b812 100644 --- a/cmd/ccm/main.go +++ b/cmd/ccm/main.go @@ -57,8 +57,6 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(kubelbv1alpha1.AddToScheme(scheme)) - utilruntime.Must(gwapiv1alpha2.Install(scheme)) - utilruntime.Must(gwapiv1.Install(scheme)) // +kubebuilder:scaffold:scheme } @@ -76,6 +74,11 @@ func main() { var kubeconfig string var useIngressClass bool var useGatewayClass bool + var disableIngressController bool + var disableGatewayController bool + var disableHTTPRouteController bool + var disableGRPCRouteController bool + var disableGatewayAPI bool if flag.Lookup("kubeconfig") == nil { flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") @@ -95,6 +98,17 @@ func main() { flag.BoolVar(&useLoadbalancerClass, "use-loadbalancer-class", false, "Use LoadBalancerClass `kubelb` to filter services. If false, all load balancer services will be managed by KubeLB.") flag.BoolVar(&useGatewayClass, "use-gateway-class", true, "Use Gateway Class `kubelb` to filter Gateway objects. Gateway should have `gatewayClassName: kubelb` set in the spec.") + flag.BoolVar(&disableIngressController, "disable-ingress-controller", false, "Disable the Ingress controller.") + flag.BoolVar(&disableGatewayAPI, "disable-gateway-api", false, "Disable the Gateway APIs and controllers.") + flag.BoolVar(&disableGatewayController, "disable-gateway-controller", false, "Disable the Gateway controller.") + flag.BoolVar(&disableHTTPRouteController, "disable-httproute-controller", false, "Disable the HTTPRoute controller.") + flag.BoolVar(&disableGRPCRouteController, "disable-grpcroute-controller", false, "Disable the GRPCRoute controller.") + + if !disableGatewayAPI { + utilruntime.Must(gwapiv1alpha2.Install(scheme)) + utilruntime.Must(gwapiv1.Install(scheme)) + } + opts := zap.Options{ Development: false, TimeEncoder: zapcore.ISO8601TimeEncoder, @@ -202,54 +216,62 @@ func main() { os.Exit(1) } - if err = (&ccm.IngressReconciler{ - Client: mgr.GetClient(), - LBClient: kubeLBMgr.GetClient(), - ClusterName: clusterName, - Log: ctrl.Log.WithName("controllers").WithName(ccm.IngressControllerName), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor(ccm.IngressControllerName), - UseIngressClass: useIngressClass, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", ccm.IngressControllerName) - os.Exit(1) + if !disableIngressController { + if err = (&ccm.IngressReconciler{ + Client: mgr.GetClient(), + LBClient: kubeLBMgr.GetClient(), + ClusterName: clusterName, + Log: ctrl.Log.WithName("controllers").WithName(ccm.IngressControllerName), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor(ccm.IngressControllerName), + UseIngressClass: useIngressClass, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", ccm.IngressControllerName) + os.Exit(1) + } } - if err = (&ccm.GatewayReconciler{ - Client: mgr.GetClient(), - LBClient: kubeLBMgr.GetClient(), - ClusterName: clusterName, - Log: ctrl.Log.WithName("controllers").WithName(ccm.GatewayControllerName), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor(ccm.GatewayControllerName), - UseGatewayClass: useGatewayClass, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", ccm.GatewayControllerName) - os.Exit(1) + if !disableGatewayController && !disableGatewayAPI { + if err = (&ccm.GatewayReconciler{ + Client: mgr.GetClient(), + LBClient: kubeLBMgr.GetClient(), + ClusterName: clusterName, + Log: ctrl.Log.WithName("controllers").WithName(ccm.GatewayControllerName), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor(ccm.GatewayControllerName), + UseGatewayClass: useGatewayClass, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", ccm.GatewayControllerName) + os.Exit(1) + } } - if err = (&ccm.HTTPRouteReconciler{ - Client: mgr.GetClient(), - LBClient: kubeLBMgr.GetClient(), - ClusterName: clusterName, - Log: ctrl.Log.WithName("controllers").WithName(ccm.GatewayHTTPRouteControllerName), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor(ccm.GatewayHTTPRouteControllerName), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", ccm.GatewayHTTPRouteControllerName) - os.Exit(1) + if !disableHTTPRouteController && !disableGatewayAPI { + if err = (&ccm.HTTPRouteReconciler{ + Client: mgr.GetClient(), + LBClient: kubeLBMgr.GetClient(), + ClusterName: clusterName, + Log: ctrl.Log.WithName("controllers").WithName(ccm.GatewayHTTPRouteControllerName), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor(ccm.GatewayHTTPRouteControllerName), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", ccm.GatewayHTTPRouteControllerName) + os.Exit(1) + } } - if err = (&ccm.GRPCRouteReconciler{ - Client: mgr.GetClient(), - LBClient: kubeLBMgr.GetClient(), - ClusterName: clusterName, - Log: ctrl.Log.WithName("controllers").WithName(ccm.GatewayGRPCRouteControllerName), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor(ccm.GatewayGRPCRouteControllerName), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", ccm.GatewayGRPCRouteControllerName) - os.Exit(1) + if !disableGRPCRouteController && !disableGatewayAPI { + if err = (&ccm.GRPCRouteReconciler{ + Client: mgr.GetClient(), + LBClient: kubeLBMgr.GetClient(), + ClusterName: clusterName, + Log: ctrl.Log.WithName("controllers").WithName(ccm.GatewayGRPCRouteControllerName), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor(ccm.GatewayGRPCRouteControllerName), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", ccm.GatewayGRPCRouteControllerName) + os.Exit(1) + } } // this is a copy and paste of SetupSignalHandler which only returns a context diff --git a/cmd/kubelb/main.go b/cmd/kubelb/main.go index bb6b7d4..d5d2ef8 100644 --- a/cmd/kubelb/main.go +++ b/cmd/kubelb/main.go @@ -36,6 +36,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) type options struct { @@ -48,6 +50,7 @@ type options struct { enableDebugMode bool namespace string enableTenantMigrationController bool + disableGatewayAPI bool } var ( @@ -73,11 +76,17 @@ func main() { flag.StringVar(&opt.namespace, "namespace", "", "The namespace where the controller will run.") flag.BoolVar(&opt.enableTenantMigrationController, "enable-tenant-migration", true, "Enables a controller that performs automated migration from namespaces to tenants") + flag.BoolVar(&opt.disableGatewayAPI, "disable-gateway-api", false, "Disable the Gateway APIs and controllers.") if flag.Lookup("kubeconfig") == nil { flag.StringVar(&opt.kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") } + if !opt.disableGatewayAPI { + utilruntime.Must(gwapiv1alpha2.Install(scheme)) + utilruntime.Must(gwapiv1.Install(scheme)) + } + if len(opt.namespace) == 0 { // Retrieve controller namespace ns, _ := os.LookupEnv("NAMESPACE") diff --git a/config/crd/bases/kubelb.k8c.io_configs.yaml b/config/crd/bases/kubelb.k8c.io_configs.yaml index 7d36922..da7fb93 100644 --- a/config/crd/bases/kubelb.k8c.io_configs.yaml +++ b/config/crd/bases/kubelb.k8c.io_configs.yaml @@ -978,6 +978,7 @@ spec: Proxy. If specified, the node must have all the indicated labels. type: object replicas: + default: 3 description: Replicas defines the number of replicas for Envoy Proxy. This field is ignored if UseDaemonset is set to true. format: int32 @@ -1086,16 +1087,18 @@ spec: type: array topology: default: shared - description: 'Topology defines the deployment topology for Envoy - Proxy. Valid values are: shared, dedicated, and global.' + description: |- + Topology defines the deployment topology for Envoy Proxy. Valid values are: shared and global. + DEPRECATION NOTICE: The value "dedicated" is deprecated and will be removed in a future release. Dedicated topology will now default to shared topology. enum: - shared - dedicated - global type: string x-kubernetes-validations: - - message: Value is immutable - rule: self == oldSelf + - message: Value is immutable and only allowed change is from + dedicated(deprecated) to shared/global + rule: self == oldSelf || (self != oldSelf && oldSelf == 'dedicated') useDaemonset: description: |- UseDaemonset defines whether Envoy Proxy will run as daemonset. By default, Envoy Proxy will run as deployment. diff --git a/config/crd/bases/kubelb.k8c.io_loadbalancers.yaml b/config/crd/bases/kubelb.k8c.io_loadbalancers.yaml index 34f6da8..85d50cb 100644 --- a/config/crd/bases/kubelb.k8c.io_loadbalancers.yaml +++ b/config/crd/bases/kubelb.k8c.io_loadbalancers.yaml @@ -16,7 +16,14 @@ spec: singular: loadbalancer scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .metadata.labels.kubelb.k8c.io/origin-name + name: OriginName + type: string + - jsonPath: .metadata.labels.kubelb.k8c.io/origin-ns + name: OriginNamespace + type: string + name: v1alpha1 schema: openAPIV3Schema: description: LoadBalancer is the Schema for the loadbalancers API diff --git a/config/crd/bases/kubelb.k8c.io_routes.yaml b/config/crd/bases/kubelb.k8c.io_routes.yaml index 4507d10..9df3edb 100644 --- a/config/crd/bases/kubelb.k8c.io_routes.yaml +++ b/config/crd/bases/kubelb.k8c.io_routes.yaml @@ -14,7 +14,17 @@ spec: singular: route scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .metadata.labels.kubelb.k8c.io/origin-name + name: OriginName + type: string + - jsonPath: .metadata.labels.kubelb.k8c.io/origin-ns + name: OriginNamespace + type: string + - jsonPath: .metadata.labels.kubelb.k8c.io/origin-resource-kind + name: OriginResource + type: string + name: v1alpha1 schema: openAPIV3Schema: description: Route is the object that represents a route in the cluster. diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 512e039..65520ce 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -13,43 +13,43 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - arm64 - - ppc64le - - s390x - - key: kubernetes.io/os - operator: In - values: - - linux + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - ppc64le + - s390x + - key: kubernetes.io/os + operator: In + values: + - linux containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--enable-leader-election" + - name: kube-rbac-proxy + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=0" + ports: + - containerPort: 8443 + protocol: TCP + name: https + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--enable-leader-election" diff --git a/docs/architecture.md b/docs/architecture.md index 38b762e..2fdbabe 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -22,7 +22,7 @@ At its core, the KubeLB manager hosts the envoy xDS server and implements the [e KubeLB manager supports three different deployment topologies for envoy proxy: -#### Dedicated +#### Dedicated (Deprecated with release v1.1.0) In this topology, the envoy proxy is deployed per load balancer service. diff --git a/go.mod b/go.mod index 9f75479..8a9c83e 100644 --- a/go.mod +++ b/go.mod @@ -10,16 +10,16 @@ require ( github.com/go-test/deep v1.1.0 github.com/golang/protobuf v1.5.4 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.33.1 + github.com/onsi/gomega v1.34.1 github.com/pkg/errors v0.9.1 go.uber.org/zap v1.27.0 google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 k8c.io/reconciler v0.5.0 - k8s.io/api v0.30.2 - k8s.io/apimachinery v0.30.2 - k8s.io/client-go v0.30.2 - k8s.io/code-generator v0.30.2 + k8s.io/api v0.30.3 + k8s.io/apimachinery v0.30.3 + k8s.io/client-go v0.30.3 + k8s.io/code-generator v0.30.3 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.18.4 sigs.k8s.io/gateway-api v1.1.0 @@ -32,7 +32,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect + github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect @@ -64,7 +64,7 @@ require ( github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/mod v0.19.0 // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect @@ -75,16 +75,16 @@ require ( golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.23.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.30.2 // indirect + k8s.io/apiextensions-apiserver v0.30.3 // indirect k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f // indirect + k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index da4d18b..a80c131 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMr github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= -github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20 h1:N+3sFI5GUjRKBi+i0TxYVST9h4Ie192jJWpHvthBBgg= +github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -103,12 +103,12 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.17.2 h1:7eMhcy3GimbsA3hEnVKdw/PQM9XN9krpKVXsZdph0/g= -github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -142,8 +142,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= -golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= @@ -197,10 +197,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY= -google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d h1:JU0iKnSg02Gmb5ZdV8nYsKEKsP6o/FGVWTrw4i1DA9A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf h1:GillM0Ef0pkZPIB+5iO6SDK+4T9pf6TpaYR6ICD5rVE= +google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:OFMYQFHJ4TM3JRlWDZhJbZfra2uqc3WLBZiaaqP4DtU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf h1:liao9UHurZLtiEwBgT9LMOnKYsHze6eA6w1KQCMVN2Q= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -229,22 +229,22 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8c.io/reconciler v0.5.0 h1:BHpelg1UfI/7oBFctqOq8sX6qzflXpl3SlvHe7e8wak= k8c.io/reconciler v0.5.0/go.mod h1:pT1+SVcVXJQeBJhpJBXQ5XW64QnKKeYTnVlQf0dGE0k= -k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= -k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= -k8s.io/apiextensions-apiserver v0.30.2 h1:l7Eue2t6QiLHErfn2vwK4KgF4NeDgjQkCXtEbOocKIE= -k8s.io/apiextensions-apiserver v0.30.2/go.mod h1:lsJFLYyK40iguuinsb3nt+Sj6CmodSI4ACDLep1rgjw= -k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= -k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= -k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= -k8s.io/code-generator v0.30.2 h1:ZY1+aGkqZVwKIyGsOzquaeZ5rSfE6wZHur8z3jQAaiw= -k8s.io/code-generator v0.30.2/go.mod h1:RQP5L67QxqgkVquk704CyvWFIq0e6RCMmLTXxjE8dVA= +k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= +k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= +k8s.io/apiextensions-apiserver v0.30.3 h1:oChu5li2vsZHx2IvnGP3ah8Nj3KyqG3kRSaKmijhB9U= +k8s.io/apiextensions-apiserver v0.30.3/go.mod h1:uhXxYDkMAvl6CJw4lrDN4CPbONkF3+XL9cacCT44kV4= +k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= +k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= +k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= +k8s.io/code-generator v0.30.3 h1:bmtnLJKagDS5f5uOEpLyJiDfIMKXGMKgOLBdde+w0Mc= +k8s.io/code-generator v0.30.3/go.mod h1:PFgBiv+miFV7TZYp+RXgROkhA+sWYZ+mtpbMLofMke8= k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 h1:NGrVE502P0s0/1hudf8zjgwki1X/TByhmAoILTarmzo= k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f h1:2sXuKesAYbRHxL3aE2PN6zX/gcJr22cjrsej+W784Tc= -k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= +k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e h1:OnKkExfhk4yxMqvBSPzUfhv3zQ96FWJ+UOZzLrAFyAo= +k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e/go.mod h1:0CVn9SVo8PeW5/JgsBZZIFmmTk5noOM8WXf2e1tCihE= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= diff --git a/hack/ci/e2e/deploy-kubelb.sh b/hack/ci/e2e/deploy-kubelb.sh index 71c74a4..59f0b69 100755 --- a/hack/ci/e2e/deploy-kubelb.sh +++ b/hack/ci/e2e/deploy-kubelb.sh @@ -29,7 +29,7 @@ kubectl wait --namespace metallb-system --for=condition=ready pod --selector=app gw=$(docker network inspect -f json kind | jq --raw-output '.[].IPAM.Config[1].Gateway' | sed -e 's/\(.*\..*\).*\..*\..*/\1/') -cat < "$ARTIFACTS/kubectl/$1.po.a.wide.out" 2>&1 ||: - kubectl -v9 get svc -A -o wide > "$ARTIFACTS/kubectl/$1.svc.a.wide.out" 2>&1 ||: - kubectl -v9 get node -o wide > "$ARTIFACTS/kubectl/$1.node.wide.out" 2>&1 ||: + kubectl -v9 get po -A -o wide > "$ARTIFACTS/kubectl/$1.po.a.wide.out" 2>&1 || : + kubectl -v9 get svc -A -o wide > "$ARTIFACTS/kubectl/$1.svc.a.wide.out" 2>&1 || : + kubectl -v9 get node -o wide > "$ARTIFACTS/kubectl/$1.node.wide.out" 2>&1 || : - kubectl -v9 get po -A -o yaml > "$ARTIFACTS/kubectl/$1.po.yaml" 2>&1 ||: - kubectl -v9 get svc -A -o yaml > "$ARTIFACTS/kubectl/$1.svc.yaml" 2>&1 ||: - kubectl -v9 get node -o yaml > "$ARTIFACTS/kubectl/$1.node.yaml" 2>&1 ||: + kubectl -v9 get po -A -o yaml > "$ARTIFACTS/kubectl/$1.po.yaml" 2>&1 || : + kubectl -v9 get svc -A -o yaml > "$ARTIFACTS/kubectl/$1.svc.yaml" 2>&1 || : + kubectl -v9 get node -o yaml > "$ARTIFACTS/kubectl/$1.node.yaml" 2>&1 || : } function dump_logs() { - if [[ ! -z "${ARTIFACTS+x}" ]]; then - echodate "Dumping logs" - mkdir -p "$ARTIFACTS/kubectl" - export KUBECONFIG="${TMPDIR}"/kubelb.kubeconfig - kubectl_get_debug_resources kubelb - export KUBECONFIG="${TMPDIR}"/tenant1.kubeconfig - kubectl_get_debug_resources tenant1 - export KUBECONFIG="${TMPDIR}"/tenant2.kubeconfig - kubectl_get_debug_resources tenant2 - - mkdir -p "$ARTIFACTS/kind_logs/{kubelb,tenant1}" - kind export logs "$ARTIFACTS/kind_logs/kubelb" --name kubelb ||: - kind export logs "$ARTIFACTS/kind_logs/tenant1" --name tenant1 ||: - kind export logs "$ARTIFACTS/kind_logs/tenant2" --name tenant2 ||: - - mkdir -p "$ARTIFACTS/pod_logs" - protokol --kubeconfig "${TMPDIR}"/kubelb.kubeconfig --flat --oneshot --output "$ARTIFACTS/pod_logs" --namespace kubelb 'kubelb-*' ||: - protokol --kubeconfig "${TMPDIR}"/kubelb.kubeconfig --flat --oneshot --output "$ARTIFACTS/pod_logs" --namespace tenant1 'kubelb-ccm-*' ||: - protokol --kubeconfig "${TMPDIR}"/kubelb.kubeconfig --flat --oneshot --output "$ARTIFACTS/pod_logs" --namespace tenant2 'kubelb-ccm-*' ||: - fi + if [[ ! -z "${ARTIFACTS+x}" ]]; then + echodate "Dumping logs" + mkdir -p "$ARTIFACTS/kubectl" + export KUBECONFIG="${TMPDIR}"/kubelb.kubeconfig + kubectl_get_debug_resources kubelb + export KUBECONFIG="${TMPDIR}"/tenant1.kubeconfig + kubectl_get_debug_resources tenant1 + export KUBECONFIG="${TMPDIR}"/tenant2.kubeconfig + kubectl_get_debug_resources tenant2 + + mkdir -p "$ARTIFACTS/kind_logs/{kubelb,tenant1}" + kind export logs "$ARTIFACTS/kind_logs/kubelb" --name kubelb || : + kind export logs "$ARTIFACTS/kind_logs/tenant1" --name tenant1 || : + kind export logs "$ARTIFACTS/kind_logs/tenant2" --name tenant2 || : + + mkdir -p "$ARTIFACTS/pod_logs" + protokol --kubeconfig "${TMPDIR}"/kubelb.kubeconfig --flat --oneshot --output "$ARTIFACTS/pod_logs" --namespace kubelb 'kubelb-*' || : + protokol --kubeconfig "${TMPDIR}"/kubelb.kubeconfig --flat --oneshot --output "$ARTIFACTS/pod_logs" --namespace tenant1 'kubelb-ccm-*' || : + protokol --kubeconfig "${TMPDIR}"/kubelb.kubeconfig --flat --oneshot --output "$ARTIFACTS/pod_logs" --namespace tenant2 'kubelb-ccm-*' || : + fi } trap dump_logs ERR @@ -68,7 +68,7 @@ trap cleanup EXIT SIGINT SIGTERM export TMPDIR=$(mktemp -d) if [[ -z "${local_ip+x}" ]]; then - local_ip=$(ip -j route show | jq -r '.[] | select(.dst == "default") | .prefsrc' | head -n1) + local_ip=$(ip -j route show | jq -r '.[] | select(.dst == "default") | .prefsrc' | head -n1) fi echodate "Pre-pulling base image in the background" @@ -78,11 +78,12 @@ echodate "Pre-pulling base image in the background" line=$(grep "as builder" $dockerfile) dockerImage=$(echo $line | awk -F'FROM | as builder' '{print $2}') echodate "Pulling image: $dockerImage" - docker pull $dockerImage &>/dev/null & + docker pull $dockerImage &> /dev/null & ) echodate "Creating kubelb kind cluster" -KUBECONFIG="${TMPDIR}"/kubelb.kubeconfig kind create cluster --retain --name kubelb --config <(cat <' - exit 1 +if [ $# -ne 1 ]; then + echo 'No cluster ID provided' + echo './create-kubelb-sa.sh ' + exit 1 fi clusterId=$1 @@ -24,7 +24,7 @@ namespace="cluster-$clusterId" kubectl create namespace "$namespace" kubectl label namespace "$namespace" kubelb.k8c.io/managed-by=kubelb -cat < EOF -} \ No newline at end of file +} + +is_containerized() { + # we're inside a Kubernetes pod/container or inside a container launched by containerize() + [ -n "${KUBERNETES_SERVICE_HOST:-}" ] || [ -n "${CONTAINERIZED:-}" ] +} + +containerize() { + local cmd="$1" + local image="${CONTAINERIZE_IMAGE:-quay.io/kubermatic/util:2.0.0}" + local gocache="${CONTAINERIZE_GOCACHE:-/tmp/.gocache}" + local gomodcache="${CONTAINERIZE_GOMODCACHE:-/tmp/.gomodcache}" + local skip="${NO_CONTAINERIZE:-}" + + # short-circuit containerize when in some cases it needs to be avoided + [ -n "$skip" ] && return + + if ! is_containerized; then + echodate "Running $cmd in a Docker container using $image..." + mkdir -p "$gocache" + mkdir -p "$gomodcache" + + exec docker run \ + -v "$PWD":/go/src/k8c.io/kubelb \ + -v "$gocache":"$gocache" \ + -v "$gomodcache":"$gomodcache" \ + -w /go/src/k8c.io/kubelb \ + -e "GOCACHE=$gocache" \ + -e "GOMODCACHE=$gomodcache" \ + -e "CONTAINERIZED=1" \ + -u "$(id -u):$(id -g)" \ + --entrypoint="$cmd" \ + --rm \ + -it \ + $image $@ + + exit $? + fi +} diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index f3d9366..ba7c59c 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - set -o errexit set -o nounset set -o pipefail @@ -28,7 +27,10 @@ OUTPUT_PKG=pkg/generated GROUP_VERSION="kubelb.k8c.io:v1alpha1" SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -CODEGEN_PKG=${CODEGEN_PKG:-$(cd "${SCRIPT_ROOT}"; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ../code-generator)} +CODEGEN_PKG=${CODEGEN_PKG:-$( + cd "${SCRIPT_ROOT}" + ls -d -1 ./vendor/k8s.io/code-generator 2> /dev/null || echo ../code-generator +)} # generate the code with: # --output-base because this script should also be able to run inside the vendor dir of @@ -40,7 +42,6 @@ bash "${CODEGEN_PKG}"/generate-groups.sh "client,lister,informer" \ --go-header-file "${SCRIPT_ROOT}"/hack/boilerplate/boilerplate.go.txt \ --output-base "$(dirname "${BASH_SOURCE[0]}")" - # The generator will create and search for k8c.io/kubelb/kubelb directory structure which is not given. # So this is a hack which takes the generated code created inside the hack folder and moves it # to the proper directory @@ -49,4 +50,4 @@ rm -r "${SCRIPT_ROOT}"/${OUTPUT_PKG}/* mv "${SCRIPT_ROOT}"/hack/${MODULE}/${OUTPUT_PKG}/* "${SCRIPT_ROOT}"/${OUTPUT_PKG}/ -rm -r "${SCRIPT_ROOT}"/hack/k8c.io \ No newline at end of file +rm -r "${SCRIPT_ROOT}"/hack/k8c.io diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh index abc9e01..d2218f7 100755 --- a/hack/verify-boilerplate.sh +++ b/hack/verify-boilerplate.sh @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - set -eu cd $(dirname $0)/.. @@ -22,4 +21,4 @@ boilerplate \ -boilerplates hack/boilerplate/ \ -exclude config \ -exclude charts \ - -exclude Makefile \ No newline at end of file + -exclude Makefile diff --git a/hack/verify-codegen.sh b/hack/verify-codegen.sh index 6dba323..6ccab06 100755 --- a/hack/verify-codegen.sh +++ b/hack/verify-codegen.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Copyright 2020 The KubeLB Authors. + +# Copyright 2024 The KubeLB Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,9 +14,46 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -euo pipefail + +cd $(dirname $0)/.. +source hack/lib.sh + +TEMPDIR=_tmp +DIFFROOTS=("api" "cmd" "config" "internal") +TMP_DIFFROOT="$TEMPDIR" + +cleanup() { + rm -rf "$TEMPDIR" +} +trap "cleanup" EXIT SIGINT + +cleanup + +mkdir -p "${TMP_DIFFROOT}" +for dir in "${DIFFROOTS[@]}"; do + cp -a "${dir}" "${TMP_DIFFROOT}/" +done + +# Update generated code +make update-codegen -set -o errexit -set -o nounset -set -o pipefail +echodate "Diffing directories against freshly generated codegen" +ret=0 +for dir in "${DIFFROOTS[@]}"; do + echodate "Checking ${dir}..." + if ! diff -Naupr "${dir}" "${TMP_DIFFROOT}/${dir}"; then + echodate "${dir} is out of date. Please run hack/update-codegen.sh" + ret=1 + else + echodate "${dir} is up to date." + fi + cp -a "${TMP_DIFFROOT}/${dir}"/* "${dir}" +done -echo "Not implemented" \ No newline at end of file +if [[ $ret -eq 0 ]]; then + echodate "All directories are up to date." +else + echodate "One or more directories are out of date. Please run hack/update-codegen.sh" + exit 1 +fi diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh new file mode 100755 index 0000000..deac23f --- /dev/null +++ b/hack/verify-licenses.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Copyright 2024 The KubeLB Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +cd $(dirname $0)/.. +source hack/lib.sh + +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 containerize ./hack/verify-licenses.sh + +go mod vendor + +echodate "Checking licenses..." +wwhrd check -q +echodate "Check successful." diff --git a/internal/controllers/ccm/gateway_controller.go b/internal/controllers/ccm/gateway_controller.go index 2e267c2..cc38d70 100644 --- a/internal/controllers/ccm/gateway_controller.go +++ b/internal/controllers/ccm/gateway_controller.go @@ -26,7 +26,6 @@ import ( kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/kubelb" - kuberneteshelper "k8c.io/kubelb/internal/kubernetes" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -36,6 +35,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -83,7 +83,7 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // Resource is marked for deletion if resource.DeletionTimestamp != nil { - if kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { + if controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { return r.cleanup(ctx, resource) } // Finalizer doesn't exist so clean up is already done @@ -95,8 +95,12 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } // Add finalizer if it doesn't exist - if !kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { - kuberneteshelper.AddFinalizer(resource, CleanupFinalizer) + if !controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + if ok := controllerutil.AddFinalizer(resource, CleanupFinalizer); !ok { + log.Error(nil, "Failed to add finalizer for the Gateway") + return ctrl.Result{Requeue: true}, nil + } + if err := r.Update(ctx, resource); err != nil { return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) } @@ -164,7 +168,7 @@ func (r *GatewayReconciler) cleanup(ctx context.Context, gateway *gwapiv1.Gatewa return reconcile.Result{}, fmt.Errorf("failed to cleanup route: %w", err) } - kuberneteshelper.RemoveFinalizer(gateway, CleanupFinalizer) + controllerutil.RemoveFinalizer(gateway, CleanupFinalizer) if err := r.Update(ctx, gateway); err != nil { return reconcile.Result{}, fmt.Errorf("failed to remove finalizer: %w", err) } diff --git a/internal/controllers/ccm/gateway_grpcroute_controller.go b/internal/controllers/ccm/gateway_grpcroute_controller.go index bb1680f..e3f609a 100644 --- a/internal/controllers/ccm/gateway_grpcroute_controller.go +++ b/internal/controllers/ccm/gateway_grpcroute_controller.go @@ -26,7 +26,6 @@ import ( kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/kubelb" - kuberneteshelper "k8c.io/kubelb/internal/kubernetes" grpcrouteHelpers "k8c.io/kubelb/internal/resources/gatewayapi/grpcroute" serviceHelpers "k8c.io/kubelb/internal/resources/service" @@ -39,6 +38,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -84,7 +84,7 @@ func (r *GRPCRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Resource is marked for deletion if resource.DeletionTimestamp != nil { - if kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { + if controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { return r.cleanup(ctx, resource) } // Finalizer doesn't exist so clean up is already done @@ -96,8 +96,8 @@ func (r *GRPCRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // Add finalizer if it doesn't exist - if !kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { - kuberneteshelper.AddFinalizer(resource, CleanupFinalizer) + if !controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + controllerutil.AddFinalizer(resource, CleanupFinalizer) if err := r.Update(ctx, resource); err != nil { return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) } @@ -190,7 +190,7 @@ func (r *GRPCRouteReconciler) cleanup(ctx context.Context, grpcRoute *gwapiv1.GR return reconcile.Result{}, fmt.Errorf("failed to cleanup route: %w", err) } - kuberneteshelper.RemoveFinalizer(grpcRoute, CleanupFinalizer) + controllerutil.RemoveFinalizer(grpcRoute, CleanupFinalizer) if err := r.Update(ctx, grpcRoute); err != nil { return reconcile.Result{}, fmt.Errorf("failed to remove finalizer: %w", err) } diff --git a/internal/controllers/ccm/gateway_httproute_controller.go b/internal/controllers/ccm/gateway_httproute_controller.go index d9e719d..5d3145e 100644 --- a/internal/controllers/ccm/gateway_httproute_controller.go +++ b/internal/controllers/ccm/gateway_httproute_controller.go @@ -26,7 +26,6 @@ import ( kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/kubelb" - kuberneteshelper "k8c.io/kubelb/internal/kubernetes" httprouteHelpers "k8c.io/kubelb/internal/resources/gatewayapi/httproute" serviceHelpers "k8c.io/kubelb/internal/resources/service" @@ -39,6 +38,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -84,7 +84,7 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Resource is marked for deletion if resource.DeletionTimestamp != nil { - if kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { + if controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { return r.cleanup(ctx, resource) } // Finalizer doesn't exist so clean up is already done @@ -96,8 +96,12 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // Add finalizer if it doesn't exist - if !kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { - kuberneteshelper.AddFinalizer(resource, CleanupFinalizer) + if !controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + if ok := controllerutil.AddFinalizer(resource, CleanupFinalizer); !ok { + log.Error(nil, "Failed to add finalizer for the HTTPRoute") + return ctrl.Result{Requeue: true}, nil + } + if err := r.Update(ctx, resource); err != nil { return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) } @@ -190,7 +194,7 @@ func (r *HTTPRouteReconciler) cleanup(ctx context.Context, httpRoute *gwapiv1.HT return reconcile.Result{}, fmt.Errorf("failed to cleanup route: %w", err) } - kuberneteshelper.RemoveFinalizer(httpRoute, CleanupFinalizer) + controllerutil.RemoveFinalizer(httpRoute, CleanupFinalizer) if err := r.Update(ctx, httpRoute); err != nil { return reconcile.Result{}, fmt.Errorf("failed to remove finalizer: %w", err) } diff --git a/internal/controllers/ccm/ingress_controller.go b/internal/controllers/ccm/ingress_controller.go index e187a6e..b02baa9 100644 --- a/internal/controllers/ccm/ingress_controller.go +++ b/internal/controllers/ccm/ingress_controller.go @@ -26,7 +26,6 @@ import ( kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/kubelb" - kuberneteshelper "k8c.io/kubelb/internal/kubernetes" ingressHelpers "k8c.io/kubelb/internal/resources/ingress" serviceHelpers "k8c.io/kubelb/internal/resources/service" @@ -40,6 +39,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -86,7 +86,7 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // Resource is marked for deletion if resource.DeletionTimestamp != nil { - if kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { + if controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { return r.cleanup(ctx, resource) } // Finalizer doesn't exist so clean up is already done @@ -98,8 +98,12 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } // Add finalizer if it doesn't exist - if !kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { - kuberneteshelper.AddFinalizer(resource, CleanupFinalizer) + if !controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + if ok := controllerutil.AddFinalizer(resource, CleanupFinalizer); !ok { + log.Error(nil, "Failed to add finalizer for the Ingress") + return ctrl.Result{Requeue: true}, nil + } + if err := r.Update(ctx, resource); err != nil { return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) } @@ -192,7 +196,7 @@ func (r *IngressReconciler) cleanup(ctx context.Context, ingress *networkingv1.I return reconcile.Result{}, fmt.Errorf("failed to cleanup route: %w", err) } - kuberneteshelper.RemoveFinalizer(ingress, CleanupFinalizer) + controllerutil.RemoveFinalizer(ingress, CleanupFinalizer) if err := r.Update(ctx, ingress); err != nil { return reconcile.Result{}, fmt.Errorf("failed to remove finalizer: %w", err) } diff --git a/internal/controllers/kubelb/envoy_cp_controller.go b/internal/controllers/kubelb/envoy_cp_controller.go index 9ce4fb9..8dac416 100644 --- a/internal/controllers/kubelb/envoy_cp_controller.go +++ b/internal/controllers/kubelb/envoy_cp_controller.go @@ -140,16 +140,7 @@ func (r *EnvoyCPReconciler) ListLoadBalancersAndRoutes(ctx context.Context, req var err error switch r.EnvoyProxyTopology { - case EnvoyProxyTopologyDedicated: - if req.Name != RequeueAllResources { - lb := kubelbv1alpha1.LoadBalancer{} - err = r.Get(ctx, req.NamespacedName, &lb) - if err != nil { - return nil, nil, err - } - loadBalancers.Items = append(loadBalancers.Items, lb) - } - case EnvoyProxyTopologyShared: + case EnvoyProxyTopologyShared, EnvoyProxyTopologyDedicated: err = r.List(ctx, &loadBalancers, client.InNamespace(req.Namespace)) if err != nil { return nil, nil, err @@ -339,10 +330,8 @@ func (r *EnvoyCPReconciler) getEnvoyProxyPodSpec(namespace, appName, snapshotNam func envoySnapshotAndAppName(topology EnvoyProxyTopology, req ctrl.Request) (string, string) { switch topology { - case EnvoyProxyTopologyShared: + case EnvoyProxyTopologyShared, EnvoyProxyTopologyDedicated: return req.Namespace, req.Namespace - case EnvoyProxyTopologyDedicated: - return fmt.Sprintf("%s-%s", req.Namespace, req.Name), req.Name case EnvoyProxyTopologyGlobal: return EnvoyGlobalCache, EnvoyGlobalCache } diff --git a/internal/controllers/kubelb/loadbalancer_controller.go b/internal/controllers/kubelb/loadbalancer_controller.go index 5fb48be..2d625f1 100644 --- a/internal/controllers/kubelb/loadbalancer_controller.go +++ b/internal/controllers/kubelb/loadbalancer_controller.go @@ -25,7 +25,6 @@ import ( "k8c.io/kubelb/internal/config" utils "k8c.io/kubelb/internal/controllers" "k8c.io/kubelb/internal/kubelb" - kuberneteshelper "k8c.io/kubelb/internal/kubernetes" portlookup "k8c.io/kubelb/internal/port-lookup" corev1 "k8s.io/api/core/v1" @@ -40,6 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -47,7 +47,7 @@ import ( ) const ( - envoyImage = "envoyproxy/envoy:distroless-v1.30.1" + envoyImage = "envoyproxy/envoy:distroless-v1.31.0" envoyProxyContainerName = "envoy-proxy" envoyResourcePattern = "envoy-%s" envoyGlobalTopologyServicePattern = "envoy-%s-%s" @@ -120,7 +120,7 @@ func (r *LoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request ) switch r.EnvoyProxyTopology { - case EnvoyProxyTopologyShared: + case EnvoyProxyTopologyShared, EnvoyProxyTopologyDedicated: err = r.List(ctx, &loadBalancers, ctrlruntimeclient.InNamespace(req.Namespace)) if err != nil { log.Error(err, "unable to fetch LoadBalancer list") @@ -136,13 +136,10 @@ func (r *LoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, err } resourceNamespace = r.Namespace - case EnvoyProxyTopologyDedicated: - loadBalancers.Items = []kubelbv1alpha1.LoadBalancer{loadBalancer} - resourceNamespace = req.Namespace } // Resource is marked for deletion. if loadBalancer.DeletionTimestamp != nil { - if kuberneteshelper.HasFinalizer(&loadBalancer, envoyProxyCleanupFinalizer) { + if controllerutil.ContainsFinalizer(&loadBalancer, envoyProxyCleanupFinalizer) { return reconcile.Result{}, r.handleEnvoyProxyCleanup(ctx, loadBalancer, resourceNamespace) } // Finalizer doesn't exist so clean up is already done @@ -150,8 +147,12 @@ func (r *LoadBalancerReconciler) Reconcile(ctx context.Context, req ctrl.Request } // Add finalizer if it doesn't exist - if !kuberneteshelper.HasFinalizer(&loadBalancer, envoyProxyCleanupFinalizer) { - kuberneteshelper.AddFinalizer(&loadBalancer, envoyProxyCleanupFinalizer) + if !controllerutil.ContainsFinalizer(&loadBalancer, envoyProxyCleanupFinalizer) { + if ok := controllerutil.AddFinalizer(&loadBalancer, envoyProxyCleanupFinalizer); !ok { + log.Error(nil, "Failed to add finalizer for the LoadBalancer") + return ctrl.Result{Requeue: true}, nil + } + if err := r.Update(ctx, &loadBalancer); err != nil { return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) } @@ -355,7 +356,7 @@ func (r *LoadBalancerReconciler) handleEnvoyProxyCleanup(ctx context.Context, lb } // Remove finalizer - kuberneteshelper.RemoveFinalizer(&lb, envoyProxyCleanupFinalizer) + controllerutil.RemoveFinalizer(&lb, envoyProxyCleanupFinalizer) // Update instance err := r.Update(ctx, &lb) diff --git a/internal/controllers/kubelb/loadbalancer_controller_test.go b/internal/controllers/kubelb/loadbalancer_controller_test.go index 4714bac..e02589d 100644 --- a/internal/controllers/kubelb/loadbalancer_controller_test.go +++ b/internal/controllers/kubelb/loadbalancer_controller_test.go @@ -52,16 +52,6 @@ var testMatrix = []struct { }, envoySnapshotName: func(lb types.NamespacedName) string { return lb.Namespace }, }, - { - topology: EnvoyProxyTopologyDedicated, - envoyProxyDeploymentName: func(lb types.NamespacedName) types.NamespacedName { - return types.NamespacedName{Name: fmt.Sprintf(envoyResourcePattern, lb.Name), Namespace: lb.Namespace} - }, - envoyProxyServiceName: func(lb types.NamespacedName) types.NamespacedName { - return types.NamespacedName{Name: fmt.Sprintf(envoyResourcePattern, lb.Name), Namespace: lb.Namespace} - }, - envoySnapshotName: func(lb types.NamespacedName) string { return fmt.Sprintf("%s-%s", lb.Namespace, lb.Name) }, - }, { topology: EnvoyProxyTopologyGlobal, envoyProxyDeploymentName: func(lb types.NamespacedName) types.NamespacedName { diff --git a/internal/controllers/kubelb/route_controller.go b/internal/controllers/kubelb/route_controller.go index 265d6b5..b4b189d 100644 --- a/internal/controllers/kubelb/route_controller.go +++ b/internal/controllers/kubelb/route_controller.go @@ -28,7 +28,6 @@ import ( kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/config" "k8c.io/kubelb/internal/kubelb" - kuberneteshelper "k8c.io/kubelb/internal/kubernetes" portlookup "k8c.io/kubelb/internal/port-lookup" serviceHelpers "k8c.io/kubelb/internal/resources/service" "k8c.io/kubelb/internal/resources/unstructured" @@ -45,6 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -93,7 +93,7 @@ func (r *RouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl // Resource is marked for deletion if resource.DeletionTimestamp != nil { - if kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { + if controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { return r.cleanup(ctx, resource) } // Finalizer doesn't exist so clean up is already done @@ -101,8 +101,11 @@ func (r *RouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl } // Add finalizer if it doesn't exist - if !kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { - kuberneteshelper.AddFinalizer(resource, CleanupFinalizer) + if !controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + if ok := controllerutil.AddFinalizer(resource, CleanupFinalizer); !ok { + log.Error(nil, "Failed to add finalizer for the Route") + return ctrl.Result{Requeue: true}, nil + } if err := r.Update(ctx, resource); err != nil { return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) } @@ -171,7 +174,7 @@ func (r *RouteReconciler) cleanup(ctx context.Context, route *kubelbv1alpha1.Rou return reconcile.Result{}, fmt.Errorf("failed to deallocate ports: %w", err) } - kuberneteshelper.RemoveFinalizer(route, CleanupFinalizer) + controllerutil.RemoveFinalizer(route, CleanupFinalizer) if err := r.Update(ctx, route); err != nil { return reconcile.Result{}, fmt.Errorf("failed to remove finalizer: %w", err) } @@ -445,7 +448,7 @@ func (r *RouteReconciler) createOrUpdateHTTPRoute(ctx context.Context, log logr. if string(ref.Name) == service.Name { ns := ref.Namespace // Corresponding service found, update the name. - if ns != nil && ns == (*gwapiv1.Namespace)(&service.Namespace) { + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) // Set the namespace to nil since all the services are created in the same namespace as the Route. object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil @@ -461,7 +464,7 @@ func (r *RouteReconciler) createOrUpdateHTTPRoute(ctx context.Context, log logr. if string(ref.Name) == service.Name { ns := ref.Namespace // Corresponding service found, update the name. - if ns != nil && ns == (*gwapiv1.Namespace)(&service.Namespace) { + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { object.Spec.Rules[i].BackendRefs[j].Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) // Set the namespace to nil since all the services are created in the same namespace as the Route. object.Spec.Rules[i].BackendRefs[j].Namespace = nil @@ -478,7 +481,7 @@ func (r *RouteReconciler) createOrUpdateHTTPRoute(ctx context.Context, log logr. if string(ref.Name) == service.Name { ns := ref.Namespace // Corresponding service found, update the name. - if ns != nil && ns == (*gwapiv1.Namespace)(&service.Namespace) { + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) // Set the namespace to nil since all the services are created in the same namespace as the Route. object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil @@ -517,6 +520,10 @@ func (r *RouteReconciler) createOrUpdateHTTPRoute(ctx context.Context, log logr. return nil } + // Required to update the object. + object.ResourceVersion = existingObject.ResourceVersion + object.UID = existingObject.UID + if err := r.Client.Update(ctx, object); err != nil { return fmt.Errorf("failed to update HTTPRoute: %w", err) } @@ -534,7 +541,7 @@ func (r *RouteReconciler) createOrUpdateGRPCRoute(ctx context.Context, log logr. if string(ref.Name) == service.Name { ns := ref.Namespace // Corresponding service found, update the name. - if ns != nil && ns == (*gwapiv1.Namespace)(&service.Namespace) { + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) // Set the namespace to nil since all the services are created in the same namespace as the Route. object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil @@ -550,7 +557,7 @@ func (r *RouteReconciler) createOrUpdateGRPCRoute(ctx context.Context, log logr. if string(ref.Name) == service.Name { ns := ref.Namespace // Corresponding service found, update the name. - if ns != nil && ns == (*gwapiv1.Namespace)(&service.Namespace) { + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { object.Spec.Rules[i].BackendRefs[j].Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) // Set the namespace to nil since all the services are created in the same namespace as the Route. object.Spec.Rules[i].BackendRefs[j].Namespace = nil @@ -567,7 +574,7 @@ func (r *RouteReconciler) createOrUpdateGRPCRoute(ctx context.Context, log logr. if string(ref.Name) == service.Name { ns := ref.Namespace // Corresponding service found, update the name. - if ns != nil && ns == (*gwapiv1.Namespace)(&service.Namespace) { + if ns == nil || ns == (*gwapiv1.Namespace)(&service.Namespace) { object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Name = gwapiv1.ObjectName(kubelb.GenerateName(r.EnvoyProxyTopology.IsGlobalTopology(), string(service.UID), service.Name, service.Namespace)) // Set the namespace to nil since all the services are created in the same namespace as the Route. object.Spec.Rules[i].Filters[j].RequestMirror.BackendRef.Namespace = nil @@ -606,6 +613,10 @@ func (r *RouteReconciler) createOrUpdateGRPCRoute(ctx context.Context, log logr. return nil } + // Required to update the object. + object.ResourceVersion = existingObject.ResourceVersion + object.UID = existingObject.UID + if err := r.Client.Update(ctx, object); err != nil { return fmt.Errorf("failed to update GRPCRoute: %w", err) } @@ -660,6 +671,10 @@ func (r *RouteReconciler) createOrUpdateIngress(ctx context.Context, log logr.Lo return nil } + // Required to update the object. + object.ResourceVersion = existingObject.ResourceVersion + object.UID = existingObject.UID + if err := r.Client.Update(ctx, object); err != nil { return fmt.Errorf("failed to update Ingress: %w", err) } diff --git a/internal/controllers/kubelb/suite_test.go b/internal/controllers/kubelb/suite_test.go index e654227..edbbd91 100644 --- a/internal/controllers/kubelb/suite_test.go +++ b/internal/controllers/kubelb/suite_test.go @@ -116,7 +116,7 @@ var _ = BeforeSuite(func() { Client: k8sManager.GetClient(), Cache: k8sManager.GetCache(), Scheme: k8sManager.GetScheme(), - EnvoyProxyTopology: EnvoyProxyTopologyDedicated, + EnvoyProxyTopology: EnvoyProxyTopologyShared, Namespace: LBNamespace, PortAllocator: portAllocator, } @@ -126,7 +126,7 @@ var _ = BeforeSuite(func() { ecpr = &EnvoyCPReconciler{ Client: k8sManager.GetClient(), EnvoyCache: envoyServer.Cache, - EnvoyProxyTopology: EnvoyProxyTopologyDedicated, + EnvoyProxyTopology: EnvoyProxyTopologyShared, EnvoyBootstrap: envoyServer.GenerateBootstrap(), Namespace: LBNamespace, PortAllocator: portAllocator, diff --git a/internal/controllers/kubelb/tenant_controller.go b/internal/controllers/kubelb/tenant_controller.go index 8672324..99fa80b 100644 --- a/internal/controllers/kubelb/tenant_controller.go +++ b/internal/controllers/kubelb/tenant_controller.go @@ -27,7 +27,6 @@ import ( kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" tenantresources "k8c.io/kubelb/internal/controllers/kubelb/resources/tenant" - kuberneteshelper "k8c.io/kubelb/internal/kubernetes" "k8c.io/reconciler/pkg/reconciling" corev1 "k8s.io/api/core/v1" @@ -41,6 +40,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -102,7 +102,7 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr // Resource is marked for deletion if resource.DeletionTimestamp != nil { - if kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { + if controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { return r.cleanup(ctx, resource) } // Finalizer doesn't exist so clean up is already done @@ -110,8 +110,11 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } // Add finalizer if it doesn't exist - if !kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { - kuberneteshelper.AddFinalizer(resource, CleanupFinalizer) + if !controllerutil.ContainsFinalizer(resource, CleanupFinalizer) { + if ok := controllerutil.AddFinalizer(resource, CleanupFinalizer); !ok { + log.Error(nil, "Failed to add finalizer for the Tenant") + return ctrl.Result{Requeue: true}, nil + } if err := r.Update(ctx, resource); err != nil { return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) } @@ -242,7 +245,7 @@ func (r *TenantReconciler) cleanup(ctx context.Context, tenant *kubelbv1alpha1.T } // Clean up is complete so remove the finalizer. - kuberneteshelper.RemoveFinalizer(tenant, CleanupFinalizer) + controllerutil.RemoveFinalizer(tenant, CleanupFinalizer) if err := r.Update(ctx, tenant); err != nil { return reconcile.Result{}, fmt.Errorf("failed to remove finalizer: %w", err) } diff --git a/internal/controllers/kubelb/tenant_migration_controller.go b/internal/controllers/kubelb/tenant_migration_controller.go index 7dcea75..b6f4366 100644 --- a/internal/controllers/kubelb/tenant_migration_controller.go +++ b/internal/controllers/kubelb/tenant_migration_controller.go @@ -25,7 +25,6 @@ import ( kubelbv1alpha1 "k8c.io/kubelb/api/kubelb.k8c.io/v1alpha1" "k8c.io/kubelb/internal/kubelb" - kuberneteshelper "k8c.io/kubelb/internal/kubernetes" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -78,14 +77,6 @@ func (r *TenantMigrationReconciler) Reconcile(ctx context.Context, req ctrl.Requ return reconcile.Result{}, nil } - // Add finalizer if it doesn't exist - if !kuberneteshelper.HasFinalizer(resource, CleanupFinalizer) { - kuberneteshelper.AddFinalizer(resource, CleanupFinalizer) - if err := r.Update(ctx, resource); err != nil { - return reconcile.Result{}, fmt.Errorf("failed to add finalizer: %w", err) - } - } - err := r.reconcile(ctx, log, resource) if err != nil { log.Error(err, "reconciling failed") diff --git a/internal/kubernetes/helper.go b/internal/kubernetes/helper.go deleted file mode 100644 index 37223d0..0000000 --- a/internal/kubernetes/helper.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2023 The KubeLB Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubernetes - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" -) - -// HasFinalizer tells if a object has all the given finalizers. -func HasFinalizer(o metav1.Object, names ...string) bool { - return sets.NewString(o.GetFinalizers()...).HasAll(names...) -} - -// AddFinalizer will add the given finalizer to the object. It uses a StringSet to avoid duplicates. -func AddFinalizer(obj metav1.Object, finalizers ...string) { - set := sets.NewString(obj.GetFinalizers()...) - set.Insert(finalizers...) - obj.SetFinalizers(set.List()) -} - -// RemoveFinalizer removes the given finalizers from the object. -func RemoveFinalizer(obj metav1.Object, toRemove ...string) { - set := sets.NewString(obj.GetFinalizers()...) - set.Delete(toRemove...) - obj.SetFinalizers(set.List()) -} diff --git a/internal/resources/gatewayapi/grpcroute/grpcroute.go b/internal/resources/gatewayapi/grpcroute/grpcroute.go index 6253f9e..d28811a 100644 --- a/internal/resources/gatewayapi/grpcroute/grpcroute.go +++ b/internal/resources/gatewayapi/grpcroute/grpcroute.go @@ -37,6 +37,8 @@ func GetServicesFromGRPCRoute(grpcroute *gwapiv1.GRPCRoute) []types.NamespacedNa if ref.Namespace != nil { serviceReference.Namespace = string(*ref.Namespace) + } else { + serviceReference.Namespace = grpcroute.Namespace } serviceReferences = append(serviceReferences, serviceReference) } @@ -51,6 +53,8 @@ func GetServicesFromGRPCRoute(grpcroute *gwapiv1.GRPCRoute) []types.NamespacedNa if ref.Namespace != nil { serviceReference.Namespace = string(*ref.Namespace) + } else { + serviceReference.Namespace = grpcroute.Namespace } serviceReferences = append(serviceReferences, serviceReference) } @@ -66,6 +70,8 @@ func GetServicesFromGRPCRoute(grpcroute *gwapiv1.GRPCRoute) []types.NamespacedNa if ref.Namespace != nil { serviceReference.Namespace = string(*ref.Namespace) + } else { + serviceReference.Namespace = grpcroute.Namespace } serviceReferences = append(serviceReferences, serviceReference) } diff --git a/internal/resources/gatewayapi/httproute/httproute.go b/internal/resources/gatewayapi/httproute/httproute.go index 9370806..34a39db 100644 --- a/internal/resources/gatewayapi/httproute/httproute.go +++ b/internal/resources/gatewayapi/httproute/httproute.go @@ -37,6 +37,8 @@ func GetServicesFromHTTPRoute(httpRoute *gwapiv1.HTTPRoute) []types.NamespacedNa if ref.Namespace != nil { serviceReference.Namespace = string(*ref.Namespace) + } else { + serviceReference.Namespace = httpRoute.Namespace } serviceReferences = append(serviceReferences, serviceReference) } @@ -51,6 +53,8 @@ func GetServicesFromHTTPRoute(httpRoute *gwapiv1.HTTPRoute) []types.NamespacedNa if ref.Namespace != nil { serviceReference.Namespace = string(*ref.Namespace) + } else { + serviceReference.Namespace = httpRoute.Namespace } serviceReferences = append(serviceReferences, serviceReference) } @@ -66,6 +70,8 @@ func GetServicesFromHTTPRoute(httpRoute *gwapiv1.HTTPRoute) []types.NamespacedNa if ref.Namespace != nil { serviceReference.Namespace = string(*ref.Namespace) + } else { + serviceReference.Namespace = httpRoute.Namespace } serviceReferences = append(serviceReferences, serviceReference) } diff --git a/internal/resources/unstructured/unstructured.go b/internal/resources/unstructured/unstructured.go index 8cba131..de24bc7 100644 --- a/internal/resources/unstructured/unstructured.go +++ b/internal/resources/unstructured/unstructured.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" ) func NormalizeUnstructured(obj *unstructured.Unstructured) *unstructured.Unstructured { @@ -64,6 +65,12 @@ func ConvertUnstructuredToObject(unstruct *unstructured.Unstructured) (client.Ob object = &networkingv1.Ingress{} case corev1.SchemeGroupVersion.WithKind("Service"): object = &corev1.Service{} + case gwapiv1.SchemeGroupVersion.WithKind("Gateway"): + object = &gwapiv1.Gateway{} + case gwapiv1.SchemeGroupVersion.WithKind("HTTPRoute"): + object = &gwapiv1.HTTPRoute{} + case gwapiv1.SchemeGroupVersion.WithKind("GRPCRoute"): + object = &gwapiv1.GRPCRoute{} } err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstruct.UnstructuredContent(), object) diff --git a/kubelb.dockerfile b/kubelb.dockerfile index 7b9d62c..cdcafdc 100644 --- a/kubelb.dockerfile +++ b/kubelb.dockerfile @@ -12,20 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM docker.io/golang:1.22.2 as builder +FROM docker.io/golang:1.22.5 as builder -WORKDIR /go/src/k8c.io/kubelb -COPY . . -RUN make build-kubelb +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download -FROM gcr.io/distroless/static:nonroot - -WORKDIR / +# Copy the go source +COPY cmd/ cmd/ +COPY api/ api/ +COPY internal/ internal/ -COPY --from=builder \ - /go/src/k8c.io/kubelb/bin/kubelb \ - /usr/local/bin/ +RUN CGO_ENABLED=0 go build -a -o kubelb cmd/kubelb/main.go +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/kubelb . USER 65532:65532 -ENTRYPOINT ["/usr/local/bin/kubelb"] +ENTRYPOINT ["/kubelb"]