diff --git a/.gitignore b/.gitignore index 8f1286c..8a6d021 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ _build .local .DS_Store /bin -cover.out \ No newline at end of file +cover.out +charts/*/Chart.lock +kubelb-*.tgz \ No newline at end of file diff --git a/.yamllint.conf b/.yamllint.conf index b110dc4..a62b3fa 100644 --- a/.yamllint.conf +++ b/.yamllint.conf @@ -10,4 +10,5 @@ ignore: | config dev vendor - .golangci.yml \ No newline at end of file + .golangci.yml + charts/* \ No newline at end of file diff --git a/Makefile b/Makefile index aca9ce3..e6aa65e 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,12 @@ else GOBIN=$(shell go env GOBIN) endif +# SED is used to allow the Makefile to be used on both Linux and macOS. +SED ?= sed +ifeq ($(shell uname), Darwin) + SED = gsed +endif + # Setting SHELL to bash allows bash commands to be executed by recipes. # Options are set to exit when a recipe line exits non-zero or a piped command fails. SHELL = /usr/bin/env bash -o pipefail @@ -59,6 +65,7 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust $(CONTROLLER_GEN) crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases $(CONTROLLER_GEN) rbac:roleName=kubelb-ccm paths="./pkg/controllers/ccm/..." output:artifacts:config=config/ccm/rbac $(CONTROLLER_GEN) rbac:roleName=kubelb paths="./pkg/controllers/kubelb/..." output:artifacts:config=config/kubelb/rbac + $(CONTROLLER_GEN) crd webhook paths="./..." output:crd:artifacts:config=charts/kubelb-manager/crds .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. @@ -198,3 +205,23 @@ $(CONTROLLER_GEN): $(LOCALBIN) envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + +HELM_DOCS ?= $(LOCALBIN)/helm-docs + +.PHONY: helm-docs +helm-docs: $(HELM_DOCS) ## Download helm-docs locally if necessary. +$(HELM_DOCS): $(LOCALBIN) + test -s $(LOCALBIN)/helm-docs || GOBIN=$(LOCALBIN) go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.11.2 + +generate-helm-docs: helm-docs + $(LOCALBIN)/helm-docs charts/ + +.PHONY: bump-chart +bump-chart: + $(SED) -i "s/^version:.*/version: $(IMAGE_TAG)/" charts/*/Chart.yaml + $(SED) -i "s/^appVersion:.*/appVersion: $(IMAGE_TAG)/" charts/*/Chart.yaml + $(SED) -i "s/tag:.*/tag: $(IMAGE_TAG)/" charts/*/values.yaml + +.PHONY: release-charts helm-docs generate-helm-docs +release-charts: bump-chart + CHART_VERSION=$(IMAGE_TAG) ./hack/release-helm-charts.sh \ No newline at end of file diff --git a/charts/kubelb-ccm/.helmignore b/charts/kubelb-ccm/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/kubelb-ccm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubelb-ccm/Chart.yaml b/charts/kubelb-ccm/Chart.yaml new file mode 100644 index 0000000..f96c04c --- /dev/null +++ b/charts/kubelb-ccm/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: kubelb-ccm +description: Helm chart for KubeLB CCM +type: application +maintainers: + - name: Kubermatic + email: support@kubermatic.com + +# Helm chart Version. +version: v1.0.0-beta.0 +# Application version to be deployed. +appVersion: v1.0.0-beta.0 \ No newline at end of file diff --git a/charts/kubelb-ccm/README.md b/charts/kubelb-ccm/README.md new file mode 100644 index 0000000..515d2b7 --- /dev/null +++ b/charts/kubelb-ccm/README.md @@ -0,0 +1,83 @@ +# kubelb-ccm + +Helm chart for KubeLB CCM. This is used to deploy the KubeLB CCM to a Kubernetes cluster. The CCM is responsible for propagating the load balancer configurations to the management cluster. + +![Version: v1.0.0-beta.0](https://img.shields.io/badge/Version-v1.0.0--beta.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.0.0-beta.0](https://img.shields.io/badge/AppVersion-v1.0.0--beta.0-informational?style=flat-square) + +## Installing the chart + +### Pre-requisites + +* Create a namespace `kubelb` for the CCM to be deployed in. +* Create imagePullSecrets for the chart to pull the image from the registry. +* The agent expects a `Secret` with a kubeconf file named `kubelb` to access the load balancer cluster. To create such run: `kubectl --namespace kubelb create secret generic kubelb-cluster --from-file=`. The name of secret cant be overriden using `.Values.kubelb.clusterSecretName` +* Update the `tenantName` in the values.yaml to a unique identifier for the tenant. This is used to identify the tenant in the manager cluster. This can be any unique string that follows [lower case RFC 1123](https://www.rfc-editor.org/rfc/rfc1123). + +At this point a minimal values.yaml should look like this: + +```yaml +imagePullSecrets: + - name: +kubelb: + clusterSecretName: kubelb-cluster + tenantName: +``` + +### Install helm chart + +Now, we can install the helm chart: + +```sh +helm registry login quay.io --username ${REGISTRY_USER} --password ${REGISTRY_PASSWORD} +helm pull oci://quay.io/kubermatic/helm-charts/kubelb-ccm --version=v1.0.0-beta.0 --untardir "kubelb-ccm" --untar +## Create and update values.yaml with the required values. +helm install kubelb-ccm kubelb-ccm --namespace kubelb -f values.yaml +``` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| autoscaling.enabled | bool | `false` | | +| autoscaling.maxReplicas | int | `10` | | +| autoscaling.minReplicas | int | `1` | | +| autoscaling.targetCPUUtilizationPercentage | int | `80` | | +| autoscaling.targetMemoryUtilizationPercentage | int | `80` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| fullnameOverride | string | `""` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"quay.io/kubermatic/kubelb-manager-ee"` | | +| image.tag | string | `"v1.0.0-beta.0"` | | +| imagePullSecrets | list | `[]` | | +| kubelb.clusterSecretName | string | `"kubelb-cluster"` | | +| kubelb.enableLeaderElection | bool | `true` | | +| kubelb.nodeAddressType | string | `"InternalIP"` | | +| kubelb.tenantName | string | `nil` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| podSecurityContext.runAsNonRoot | bool | `true` | | +| podSecurityContext.seccompProfile.type | string | `"RuntimeDefault"` | | +| rbac.allowLeaderElectionRole | bool | `true` | | +| rbac.allowMetricsReaderRole | bool | `true` | | +| rbac.allowProxyRole | bool | `true` | | +| rbac.enabled | bool | `true` | | +| replicaCount | int | `1` | | +| resources.limits.cpu | string | `"100m"` | | +| resources.limits.memory | string | `"128Mi"` | | +| resources.requests.cpu | string | `"100m"` | | +| resources.requests.memory | string | `"128Mi"` | | +| securityContext.allowPrivilegeEscalation | bool | `false` | | +| securityContext.capabilities.drop[0] | string | `"ALL"` | | +| securityContext.runAsUser | int | `65532` | | +| service.port | int | `8443` | | +| service.protocol | string | `"TCP"` | | +| service.type | string | `"ClusterIP"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| serviceMonitor.enabled | bool | `false` | | +| tolerations | list | `[]` | | diff --git a/charts/kubelb-ccm/README.md.gotmpl b/charts/kubelb-ccm/README.md.gotmpl new file mode 100644 index 0000000..1469ed2 --- /dev/null +++ b/charts/kubelb-ccm/README.md.gotmpl @@ -0,0 +1,39 @@ +# kubelb-ccm + +Helm chart for KubeLB CCM. This is used to deploy the KubeLB CCM to a Kubernetes cluster. The CCM is responsible for propagating the load balancer configurations to the management cluster. + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +## Installing the chart + +### Pre-requisites + +* Create a namespace `kubelb` for the CCM to be deployed in. +* Create imagePullSecrets for the chart to pull the image from the registry. +* The agent expects a `Secret` with a kubeconf file named `kubelb` to access the load balancer cluster. To create such run: `kubectl --namespace kubelb create secret generic kubelb-cluster --from-file=`. The name of secret cant be overriden using `.Values.kubelb.clusterSecretName` +* Update the `tenantName` in the values.yaml to a unique identifier for the tenant. This is used to identify the tenant in the manager cluster. This can be any unique string that follows [lower case RFC 1123](https://www.rfc-editor.org/rfc/rfc1123). + +At this point a minimal values.yaml should look like this: + +```yaml +imagePullSecrets: + - name: +kubelb: + clusterSecretName: kubelb-cluster + tenantName: +``` + +### Install helm chart + +Now, we can install the helm chart: + +```sh +helm registry login quay.io --username ${REGISTRY_USER} --password ${REGISTRY_PASSWORD} +helm pull oci://quay.io/kubermatic/helm-charts/kubelb-ccm --version=v1.0.0-beta.0 --untardir "kubelb-ccm" --untar +## Create and update values.yaml with the required values. +helm install kubelb-ccm kubelb-ccm --namespace kubelb -f values.yaml +``` + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} diff --git a/charts/kubelb-ccm/templates/_helpers.tpl b/charts/kubelb-ccm/templates/_helpers.tpl new file mode 100644 index 0000000..5319a40 --- /dev/null +++ b/charts/kubelb-ccm/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "kubelb-ccm.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kubelb-ccm.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kubelb-ccm.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "kubelb-ccm.labels" -}} +helm.sh/chart: {{ include "kubelb-ccm.chart" . }} +{{ include "kubelb-ccm.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kubelb-ccm.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kubelb-ccm.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kubelb-ccm.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "kubelb-ccm.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/kubelb-ccm/templates/clusterrole.yaml b/charts/kubelb-ccm/templates/clusterrole.yaml new file mode 100644 index 0000000..aa7e57e --- /dev/null +++ b/charts/kubelb-ccm/templates/clusterrole.yaml @@ -0,0 +1,67 @@ + +{{- if .Values.rbac.enabled -}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubelb-ccm.fullname" . }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - get + - patch + - update +--- +{{- if .Values.rbac.allowProxyRole }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubelb-ccm.fullname" . }}-proxy +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +{{- end }} +--- +{{- if .Values.rbac.allowMetricsReaderRole }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubelb-ccm.fullname" . }}-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubelb-ccm/templates/clusterrolebinding.yaml b/charts/kubelb-ccm/templates/clusterrolebinding.yaml new file mode 100644 index 0000000..844865b --- /dev/null +++ b/charts/kubelb-ccm/templates/clusterrolebinding.yaml @@ -0,0 +1,45 @@ +{{- if .Values.rbac.enabled -}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "kubelb-ccm.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "kubelb-ccm.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "kubelb-ccm.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +{{- if .Values.rbac.allowProxyRole }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "kubelb-ccm.fullname" . }}-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "kubelb-ccm.fullname" . }}-proxy +subjects: +- kind: ServiceAccount + name: {{ include "kubelb-ccm.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} +--- +{{- if .Values.rbac.allowMetricsReaderRole }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "kubelb-ccm.fullname" . }}-metrics-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "kubelb-ccm.fullname" . }}-metrics-reader +subjects: +- kind: ServiceAccount + name: {{ include "kubelb-ccm.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubelb-ccm/templates/deployment.yaml b/charts/kubelb-ccm/templates/deployment.yaml new file mode 100644 index 0000000..2935993 --- /dev/null +++ b/charts/kubelb-ccm/templates/deployment.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubelb-ccm.fullname" . }} + labels: + {{- include "kubelb-ccm.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "kubelb-ccm.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "kubelb-ccm.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "kubelb-ccm.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=0 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 + imagePullPolicy: {{ .Values.image.pullPolicy }} + name: kube-rbac-proxy + ports: + - protocol: TCP + containerPort: 8443 + name: https + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --enable-leader-election={{ .Values.kubelb.enableLeaderElection }} + - --node-address-type={{ default "InternalIP" .Values.kubelb.nodeAddressType }} + - --cluster-name={{ required "A valid .Values.kubelb.tenantName to specify the tenant name is required!" .Values.kubelb.tenantName }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - mountPath: /home/nonroot/.kube + name: kubelb-cluster + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: kubelb-cluster + secret: + secretName: "{{ default "kubelb-cluster" .Values.kubelb.clusterSecretName }}" + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/kubelb-ccm/templates/hpa.yaml b/charts/kubelb-ccm/templates/hpa.yaml new file mode 100644 index 0000000..2739b91 --- /dev/null +++ b/charts/kubelb-ccm/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "kubelb-ccm.fullname" . }} + labels: + {{- include "kubelb-ccm.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "kubelb-ccm.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/kubelb-ccm/templates/role.yaml b/charts/kubelb-ccm/templates/role.yaml new file mode 100644 index 0000000..c9895af --- /dev/null +++ b/charts/kubelb-ccm/templates/role.yaml @@ -0,0 +1,44 @@ +{{- if .Values.rbac.enabled -}} +{{- if .Values.rbac.allowLeaderElectionRole }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "kubelb-ccm.fullname" . }}-leader-election + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelb-ccm.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubelb-ccm/templates/rolebinding.yaml b/charts/kubelb-ccm/templates/rolebinding.yaml new file mode 100644 index 0000000..84d7648 --- /dev/null +++ b/charts/kubelb-ccm/templates/rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.enabled -}} +{{- if .Values.rbac.allowLeaderElectionRole }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "kubelb-ccm.fullname" . }}-leader-election + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelb-ccm.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "kubelb-ccm.fullname" . }}-leader-election +subjects: +- kind: ServiceAccount + name: {{ include "kubelb-ccm.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubelb-ccm/templates/service.yaml b/charts/kubelb-ccm/templates/service.yaml new file mode 100644 index 0000000..a872c35 --- /dev/null +++ b/charts/kubelb-ccm/templates/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubelb-ccm.fullname" . }}-metrics-service + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelb-ccm.labels" . | nindent 4 }} +spec: + ports: + - name: https + port: 8443 + targetPort: https + selector: + {{- include "kubelb-ccm.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/charts/kubelb-ccm/templates/serviceaccount.yaml b/charts/kubelb-ccm/templates/serviceaccount.yaml new file mode 100644 index 0000000..8a33a97 --- /dev/null +++ b/charts/kubelb-ccm/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubelb-ccm.serviceAccountName" . }} + labels: + {{- include "kubelb-ccm.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/kubelb-ccm/templates/servicemonitor.yaml b/charts/kubelb-ccm/templates/servicemonitor.yaml new file mode 100644 index 0000000..8285d05 --- /dev/null +++ b/charts/kubelb-ccm/templates/servicemonitor.yaml @@ -0,0 +1,20 @@ +{{- if .Values.serviceMonitor.enabled -}} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "kubelb-ccm.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelb-ccm.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "kubelb-ccm.selectorLabels" . | nindent 4 }} + endpoints: + - port: metrics + path: /metrics + namespaceSelector: + matchNames: + - '{{ .Release.Namespace }}' +{{- end }} \ No newline at end of file diff --git a/charts/kubelb-ccm/templates/tests/test-connection.yaml b/charts/kubelb-ccm/templates/tests/test-connection.yaml new file mode 100644 index 0000000..b292e62 --- /dev/null +++ b/charts/kubelb-ccm/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "kubelb-ccm.fullname" . }}-test-connection" + labels: + {{- include "kubelb-ccm.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "kubelb-ccm.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/kubelb-ccm/values.yaml b/charts/kubelb-ccm/values.yaml new file mode 100644 index 0000000..035b5ad --- /dev/null +++ b/charts/kubelb-ccm/values.yaml @@ -0,0 +1,75 @@ +replicaCount: 1 + +image: + repository: quay.io/kubermatic/kubelb-manager-ee + pullPolicy: IfNotPresent + tag: v1.0.0-beta.0 +imagePullSecrets: [] + +kubelb: + tenantName: null + enableLeaderElection: true + nodeAddressType: InternalIP + clusterSecretName: kubelb-cluster + +extraVolumes: [] +extraVolumeMounts: [] + +nameOverride: "" +fullnameOverride: "" +podAnnotations: {} +podLabels: {} + +## RBAC +serviceAccount: + create: true + annotations: {} + name: "" + +rbac: + enabled: true + allowProxyRole: true + allowMetricsReaderRole: true + allowLeaderElectionRole: true + +# Monitoring Configuration +serviceMonitor: + enabled: false + +podSecurityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsUser: 65532 + +service: + type: ClusterIP + port: 8443 + protocol: TCP + +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 10 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/charts/kubelb-manager/.helmignore b/charts/kubelb-manager/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/kubelb-manager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubelb-manager/Chart.yaml b/charts/kubelb-manager/Chart.yaml new file mode 100644 index 0000000..f20eaff --- /dev/null +++ b/charts/kubelb-manager/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: kubelb-manager +description: Helm chart for KubeLB Manager +type: application +maintainers: + - name: Kubermatic + email: support@kubermatic.com + +# Helm chart Version. +version: v1.0.0-beta.0 +# Application version to be deployed. +appVersion: v1.0.0-beta.0 \ No newline at end of file diff --git a/charts/kubelb-manager/README.md b/charts/kubelb-manager/README.md new file mode 100644 index 0000000..d6f3230 --- /dev/null +++ b/charts/kubelb-manager/README.md @@ -0,0 +1,78 @@ +# kubelb-manager + +Helm chart for KubeLB Manager. This is used to deploy the KubeLB CCM to a Kubernetes cluster. The CCM is responsible for propagating the load balancer configurations to the management cluster. + +![Version: v1.0.0-beta.0](https://img.shields.io/badge/Version-v1.0.0--beta.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.0.0-beta.0](https://img.shields.io/badge/AppVersion-v1.0.0--beta.0-informational?style=flat-square) + +## Installing the chart + +### Pre-requisites + +* Create a namespace `kubelb` for the CCM to be deployed in. +* Create imagePullSecrets for the chart to pull the image from the registry. + +At this point a minimal values.yaml should look like this: + +```yaml +imagePullSecrets: + - name: +``` + +### Install helm chart + +Now, we can install the helm chart: + +```sh +helm registry login quay.io --username ${REGISTRY_USER} --password ${REGISTRY_PASSWORD} +helm pull oci://quay.io/kubermatic/helm-charts/kubelb-manager --version=v1.0.0-beta.0 --untardir "kubelb-manager" --untar +## Create and update values.yaml with the required values. +helm install kubelb-manager kubelb-manager --namespace kubelb -f values.yaml +``` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| autoscaling.enabled | bool | `false` | | +| autoscaling.maxReplicas | int | `10` | | +| autoscaling.minReplicas | int | `1` | | +| autoscaling.targetCPUUtilizationPercentage | int | `80` | | +| autoscaling.targetMemoryUtilizationPercentage | int | `80` | | +| fullnameOverride | string | `""` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"quay.io/kubermatic/kubelb-manager-ee"` | | +| image.tag | string | `"v1.0.0-beta.0"` | | +| imagePullSecrets | list | `[]` | | +| kubelb.debug | bool | `false` | | +| kubelb.enableLeaderElection | bool | `true` | | +| kubelb.envoyProxyReplicas | int | `3` | | +| kubelb.envoyProxySinglePodPerNode | bool | `true` | | +| kubelb.envoyProxyTopology | string | `"shared"` | | +| kubelb.envoyProxyUseDaemonset | bool | `false` | | +| nameOverride | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| podSecurityContext.runAsNonRoot | bool | `true` | | +| podSecurityContext.seccompProfile.type | string | `"RuntimeDefault"` | | +| rbac.allowLeaderElectionRole | bool | `true` | | +| rbac.allowMetricsReaderRole | bool | `true` | | +| rbac.allowProxyRole | bool | `true` | | +| rbac.enabled | bool | `true` | | +| replicaCount | int | `1` | | +| resources.limits.cpu | string | `"100m"` | | +| resources.limits.memory | string | `"128Mi"` | | +| resources.requests.cpu | string | `"100m"` | | +| resources.requests.memory | string | `"128Mi"` | | +| securityContext.allowPrivilegeEscalation | bool | `false` | | +| securityContext.capabilities.drop[0] | string | `"ALL"` | | +| securityContext.runAsUser | int | `65532` | | +| service.port | int | `8001` | | +| service.protocol | string | `"TCP"` | | +| service.type | string | `"ClusterIP"` | | +| serviceAccount.annotations | object | `{}` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `""` | | +| serviceMonitor.enabled | bool | `false` | | +| tolerations | list | `[]` | | diff --git a/charts/kubelb-manager/README.md.gotmpl b/charts/kubelb-manager/README.md.gotmpl new file mode 100644 index 0000000..b3c32e2 --- /dev/null +++ b/charts/kubelb-manager/README.md.gotmpl @@ -0,0 +1,34 @@ +# kubelb-manager + +Helm chart for KubeLB Manager. This is used to deploy the KubeLB CCM to a Kubernetes cluster. The CCM is responsible for propagating the load balancer configurations to the management cluster. + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +## Installing the chart + +### Pre-requisites + +* Create a namespace `kubelb` for the CCM to be deployed in. +* Create imagePullSecrets for the chart to pull the image from the registry. + +At this point a minimal values.yaml should look like this: + +```yaml +imagePullSecrets: + - name: +``` + +### Install helm chart + +Now, we can install the helm chart: + +```sh +helm registry login quay.io --username ${REGISTRY_USER} --password ${REGISTRY_PASSWORD} +helm pull oci://quay.io/kubermatic/helm-charts/kubelb-manager --version=v1.0.0-beta.0 --untardir "kubelb-manager" --untar +## Create and update values.yaml with the required values. +helm install kubelb-manager kubelb-manager --namespace kubelb -f values.yaml +``` + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} diff --git a/charts/kubelb-manager/crds/kubelb.k8c.io_loadbalancers.yaml b/charts/kubelb-manager/crds/kubelb.k8c.io_loadbalancers.yaml new file mode 100644 index 0000000..944d8a0 --- /dev/null +++ b/charts/kubelb-manager/crds/kubelb.k8c.io_loadbalancers.yaml @@ -0,0 +1,290 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: loadbalancers.kubelb.k8c.io +spec: + group: kubelb.k8c.io + names: + kind: LoadBalancer + listKind: LoadBalancerList + plural: loadbalancers + shortNames: + - lb + singular: loadbalancer + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: LoadBalancer is the Schema for the loadbalancers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: LoadBalancerSpec defines the desired state of LoadBalancer + properties: + endpoints: + description: Sets of addresses and ports that comprise an exposed + user service on a cluster. + items: + description: "LoadBalancerEndpoints is a group of addresses with + a common set of ports. The expanded set of endpoints is the Cartesian + product of Addresses x Ports. For example, given: \n { Addresses: + [{\"ip\": \"10.10.1.1\"}, {\"ip\": \"10.10.2.2\"}], Ports: [{\"name\": + \"a\", \"port\": 8675}, {\"name\": \"b\", \"port\": 309}] } \n + The resulting set of endpoints can be viewed as: \n a: [ 10.10.1.1:8675, + 10.10.2.2:8675 ], b: [ 10.10.1.1:309, 10.10.2.2:309 ]" + properties: + addresses: + description: IP addresses which offer the related ports that + are marked as ready. These endpoints should be considered + safe for load balancers and clients to utilize. + items: + description: EndpointAddress is a tuple that describes single + IP address. + properties: + hostname: + description: The Hostname of this endpoint + type: string + ip: + description: The IP of this endpoint. May not be loopback + (127.0.0.0/8), link-local (169.254.0.0/16), or link-local + multicast ((224.0.0.0/24). + minLength: 7 + type: string + required: + - ip + type: object + minItems: 1 + type: array + ports: + description: Port numbers available on the related IP addresses. + items: + description: EndpointPort is a tuple that describes a single + port. + properties: + name: + description: The name of this port. This must match the + 'name' field in the corresponding ServicePort. Must + be a DNS_LABEL. Optional only if one port is defined. + type: string + port: + description: The port number of the endpoint. + format: int32 + type: integer + protocol: + default: TCP + description: The IP protocol for this port. Must be TCP. + Default is TCP. + enum: + - TCP + type: string + required: + - port + type: object + minItems: 1 + type: array + type: object + minItems: 1 + type: array + ports: + description: The list of ports that are exposed by the load balancer + service. only needed for layer 4 + items: + description: LoadBalancerPort contains information on service's + port. + properties: + name: + description: The name of this port within the service. This + must be a DNS_LABEL. All ports within a Spec must have unique + names. When considering the endpoints for a Service, this + must match the 'name' field in the EndpointPort. Optional + if only one ServicePort is defined on this service. + type: string + port: + description: The port that will be exposed by the LoadBalancer. + format: int32 + type: integer + protocol: + default: TCP + description: The IP protocol for this port. Supports "TCP". + Default is TCP. + enum: + - TCP + type: string + required: + - port + type: object + type: array + type: + default: ClusterIP + description: 'type determines how the Service is exposed. Defaults + to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, + and LoadBalancer. "ExternalName" maps to the specified externalName. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that + is not specified, by manual construction of an Endpoints object. + If clusterIP is "None", no virtual IP is allocated and the endpoints + are published as a set of endpoints rather than a stable IP. "NodePort" + builds on ClusterIP and allocates a port on every node which routes + to the clusterIP. "LoadBalancer" builds on NodePort and creates + an external load-balancer (if supported in the current cloud) which + routes to the clusterIP. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + status: + description: LoadBalancerStatus defines the observed state of LoadBalancer + properties: + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer, + if one is present. + properties: + ingress: + description: Ingress is a list containing ingress points for the + load-balancer. Traffic intended for the service should be sent + to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a + load-balancer ingress point: traffic intended for the service + should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points + that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points + that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports + If used, every port defined in the service should have + an entry in it + items: + properties: + error: + description: 'Error is to record the problem with + the service port The format of the error shall comply + with the following rules: - built-in error values + shall be specified in this file and those shall + use CamelCase names - cloud provider specific error + values must have names that comply with the format + foo.example.com/CamelCase. --- The regex it matches + is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service + port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service + port of which status is recorded here The supported + values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + service: + description: Service contains the current status of the LB service. + properties: + ports: + items: + description: ServicePort contains information on service's port. + properties: + appProtocol: + description: "The application protocol for this port. This + is used as a hint for implementations to offer richer + behavior for protocols that they understand. This field + follows standard Kubernetes label syntax. Valid values + are either: \n * Un-prefixed protocol names - reserved + for IANA standard service names (as per RFC-6335 and https://www.iana.org/assignments/service-names). + \n * Kubernetes-defined prefixed names: * 'kubernetes.io/h2c' + - HTTP/2 over cleartext as described in https://www.rfc-editor.org/rfc/rfc7540 + * 'kubernetes.io/ws' - WebSocket over cleartext as described + in https://www.rfc-editor.org/rfc/rfc6455 * 'kubernetes.io/wss' + - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 + \n * Other protocols should use implementation-defined + prefixed names such as mycompany.com/my-custom-protocol." + type: string + name: + description: The name of this port within the service. This + must be a DNS_LABEL. All ports within a ServiceSpec must + have unique names. When considering the endpoints for + a Service, this must match the 'name' field in the EndpointPort. + Optional if only one ServicePort is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this service + is exposed when type is NodePort or LoadBalancer. Usually + assigned by the system. If a value is specified, in-range, + and not in use it will be used, otherwise the operation + will fail. If not specified, a port will be allocated + if this Service requires one. If this field is specified + when creating a Service which does not need it, creation + will fail. This field will be wiped when updating a Service + to no longer need it (e.g. changing type from NodePort + to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this service. + format: int32 + type: integer + protocol: + default: TCP + description: The IP protocol for this port. Supports "TCP", + "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access on the + pods targeted by the service. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. If this is + a string, it will be looked up as a named port in the + target Pod''s container ports. If this is not specified, + the value of the ''port'' field is used (an identity map). + This field is ignored for services with clusterIP=None, + and should be omitted or set equal to the ''port'' field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + x-kubernetes-int-or-string: true + upstreamTargetPort: + format: int32 + type: integer + required: + - port + - upstreamTargetPort + type: object + type: array + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/kubelb-manager/templates/_helpers.tpl b/charts/kubelb-manager/templates/_helpers.tpl new file mode 100644 index 0000000..8f3b32a --- /dev/null +++ b/charts/kubelb-manager/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "kubelb-manager.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kubelb-manager.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kubelb-manager.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "kubelb-manager.labels" -}} +helm.sh/chart: {{ include "kubelb-manager.chart" . }} +{{ include "kubelb-manager.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kubelb-manager.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kubelb-manager.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kubelb-manager.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "kubelb-manager.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/kubelb-manager/templates/clusterrole.yaml b/charts/kubelb-manager/templates/clusterrole.yaml new file mode 100644 index 0000000..9202ea5 --- /dev/null +++ b/charts/kubelb-manager/templates/clusterrole.yaml @@ -0,0 +1,105 @@ + +{{- if .Values.rbac.enabled -}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubelb-manager.fullname" . }} +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubelb.k8c.io + resources: + - loadbalancers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubelb.k8c.io + resources: + - loadbalancers/status + verbs: + - get + - patch + - update +--- +{{- if .Values.rbac.allowProxyRole }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubelb-manager.fullname" . }}-proxy +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +{{- end }} +--- +{{- if .Values.rbac.allowMetricsReaderRole }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "kubelb-manager.fullname" . }}-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubelb-manager/templates/clusterrolebinding.yaml b/charts/kubelb-manager/templates/clusterrolebinding.yaml new file mode 100644 index 0000000..9f1c267 --- /dev/null +++ b/charts/kubelb-manager/templates/clusterrolebinding.yaml @@ -0,0 +1,45 @@ +{{- if .Values.rbac.enabled -}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "kubelb-manager.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "kubelb-manager.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "kubelb-manager.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +{{- if .Values.rbac.allowProxyRole }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "kubelb-manager.fullname" . }}-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "kubelb-manager.fullname" . }}-proxy +subjects: +- kind: ServiceAccount + name: {{ include "kubelb-manager.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} +--- +{{- if .Values.rbac.allowMetricsReaderRole }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "kubelb-manager.fullname" . }}-metrics-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "kubelb-manager.fullname" . }}-metrics-reader +subjects: +- kind: ServiceAccount + name: {{ include "kubelb-manager.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubelb-manager/templates/deployment.yaml b/charts/kubelb-manager/templates/deployment.yaml new file mode 100644 index 0000000..666da5b --- /dev/null +++ b/charts/kubelb-manager/templates/deployment.yaml @@ -0,0 +1,100 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubelb-manager.fullname" . }} + labels: + {{- include "kubelb-manager.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "kubelb-manager.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "kubelb-manager.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "kubelb-manager.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - args: + - --secure-listen-address=0.0.0.0:8443 + - --upstream=http://127.0.0.1:8080/ + - --logtostderr=true + - --v=0 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 + imagePullPolicy: {{ .Values.image.pullPolicy }} + name: kube-rbac-proxy + ports: + - protocol: TCP + containerPort: 8443 + name: https + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --enable-leader-election={{ .Values.kubelb.enableLeaderElection }} + - --envoy-proxy-topology={{ .Values.kubelb.envoyProxyTopology }} + - --envoy-proxy-replicas={{ .Values.kubelb.envoyProxyReplicas }} + - --envoy-proxy-single-pod-per-node={{ .Values.kubelb.envoyProxySinglePodPerNode }} + - --envoy-proxy-use-daemonset={{ .Values.kubelb.envoyProxyUseDaemonset }} + - --debug={{ .Values.kubelb.debug }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/kubelb-manager/templates/hpa.yaml b/charts/kubelb-manager/templates/hpa.yaml new file mode 100644 index 0000000..8ffa061 --- /dev/null +++ b/charts/kubelb-manager/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "kubelb-manager.fullname" . }} + labels: + {{- include "kubelb-manager.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "kubelb-manager.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/kubelb-manager/templates/role.yaml b/charts/kubelb-manager/templates/role.yaml new file mode 100644 index 0000000..47f6eb5 --- /dev/null +++ b/charts/kubelb-manager/templates/role.yaml @@ -0,0 +1,44 @@ +{{- if .Values.rbac.enabled -}} +{{- if .Values.rbac.allowLeaderElectionRole }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "kubelb-manager.fullname" . }}-leader-election + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelb-manager.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubelb-manager/templates/rolebinding.yaml b/charts/kubelb-manager/templates/rolebinding.yaml new file mode 100644 index 0000000..942c89d --- /dev/null +++ b/charts/kubelb-manager/templates/rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.enabled -}} +{{- if .Values.rbac.allowLeaderElectionRole }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "kubelb-manager.fullname" . }}-leader-election + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelb-manager.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "kubelb-manager.fullname" . }}-leader-election +subjects: +- kind: ServiceAccount + name: {{ include "kubelb-manager.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubelb-manager/templates/service.yaml b/charts/kubelb-manager/templates/service.yaml new file mode 100644 index 0000000..b5ac209 --- /dev/null +++ b/charts/kubelb-manager/templates/service.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Service +metadata: + name: envoycp + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelb-manager.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: 8001 + protocol: TCP + name: http + selector: + {{- include "kubelb-manager.selectorLabels" . | nindent 4 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubelb-manager.fullname" . }}-metrics-service + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelb-manager.labels" . | nindent 4 }} +spec: + ports: + - name: https + port: 8443 + targetPort: https + selector: + {{- include "kubelb-manager.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/charts/kubelb-manager/templates/serviceaccount.yaml b/charts/kubelb-manager/templates/serviceaccount.yaml new file mode 100644 index 0000000..25963fe --- /dev/null +++ b/charts/kubelb-manager/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kubelb-manager.serviceAccountName" . }} + labels: + {{- include "kubelb-manager.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/kubelb-manager/templates/servicemonitor.yaml b/charts/kubelb-manager/templates/servicemonitor.yaml new file mode 100644 index 0000000..06fc965 --- /dev/null +++ b/charts/kubelb-manager/templates/servicemonitor.yaml @@ -0,0 +1,20 @@ +{{- if .Values.serviceMonitor.enabled -}} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "kubelb-manager.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubelb-manager.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "kubelb-manager.selectorLabels" . | nindent 4 }} + endpoints: + - port: metrics + path: /metrics + namespaceSelector: + matchNames: + - '{{ .Release.Namespace }}' +{{- end }} \ No newline at end of file diff --git a/charts/kubelb-manager/templates/tests/test-connection.yaml b/charts/kubelb-manager/templates/tests/test-connection.yaml new file mode 100644 index 0000000..ea3d62d --- /dev/null +++ b/charts/kubelb-manager/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "kubelb-manager.fullname" . }}-test-connection" + labels: + {{- include "kubelb-manager.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "kubelb-manager.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/kubelb-manager/values.yaml b/charts/kubelb-manager/values.yaml new file mode 100644 index 0000000..06a0250 --- /dev/null +++ b/charts/kubelb-manager/values.yaml @@ -0,0 +1,74 @@ +replicaCount: 1 + +image: + repository: quay.io/kubermatic/kubelb-manager-ee + pullPolicy: IfNotPresent + tag: v1.0.0-beta.0 +imagePullSecrets: [] + +kubelb: + enableLeaderElection: true + envoyProxyTopology: shared + envoyProxyReplicas: 3 + envoyProxySinglePodPerNode: true + envoyProxyUseDaemonset: false + debug: false + +nameOverride: "" +fullnameOverride: "" +podAnnotations: {} +podLabels: {} + +## RBAC +serviceAccount: + create: true + annotations: {} + name: "" + +rbac: + enabled: true + allowProxyRole: true + allowMetricsReaderRole: true + allowLeaderElectionRole: true + +# Monitoring Configuration +serviceMonitor: + enabled: false + +podSecurityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsUser: 65532 + +service: + type: ClusterIP + port: 8001 + protocol: TCP + +resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 10 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 49acdc2..5c1f7ce 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -30,8 +30,6 @@ patchesStrategicMerge: # endpoint w/o any authn/z, please comment the following line. - manager_auth_proxy_patch.yaml - - # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml #- manager_webhook_patch.yaml diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index ea52de4..512e039 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -31,7 +31,7 @@ spec: capabilities: drop: - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0 + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 args: - "--secure-listen-address=0.0.0.0:8443" - "--upstream=http://127.0.0.1:8080/" @@ -52,4 +52,4 @@ spec: args: - "--health-probe-bind-address=:8081" - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" + - "--enable-leader-election" diff --git a/config/kubelb/kustomization.yaml b/config/kubelb/kustomization.yaml index 00c9845..eb67dc6 100644 --- a/config/kubelb/kustomization.yaml +++ b/config/kubelb/kustomization.yaml @@ -1,6 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization - namespace: kubelb resources: - ../crd diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 2301724..ec69a9d 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -10,7 +10,7 @@ resources: # Comment the following 4 lines if you want to disable # the auth proxy (https://github.com/brancz/kube-rbac-proxy) # which protects your /metrics endpoint. -# - auth_proxy_service.yaml -# - auth_proxy_role.yaml -# - auth_proxy_role_binding.yaml -# - auth_proxy_client_clusterrole.yaml +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/hack/release-helm-charts.sh b/hack/release-helm-charts.sh new file mode 100755 index 0000000..f0aa86e --- /dev/null +++ b/hack/release-helm-charts.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Copyright 2023 The KubeLB Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +cd $(dirname $0)/.. +source hack/lib.sh + +REGISTRY_HOST="${REGISTRY_HOST:-quay.io}" +REPOSITORY_PREFIX="${REPOSITORY_PREFIX:-kubermatic/helm-charts}" + +if [ -z "${VAULT_ADDR:-}" ]; then + export VAULT_ADDR=https://vault.kubermatic.com/ +fi + +REGISTRY_USER="${REGISTRY_USER:-$(vault kv get -field=username dev/kubermatic-quay.io)}" +REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-$(vault kv get -field=password dev/kubermatic-quay.io)}" + +echo ${REGISTRY_PASSWORD} | helm registry login ${REGISTRY_HOST} --username ${REGISTRY_USER} --password-stdin + +# Package and publish charts +MANAGER="kubelb-manager" +CCM="kubelb-ccm" +CHART_PACKAGE_MANAGER="${MANAGER}-${CHART_VERSION}.tgz" +CHART_PACKAGE_CCM="${CCM}-${CHART_VERSION}.tgz" + +echodate "Packaging helm charts ${CHART_PACKAGE_MANAGER} and ${CHART_PACKAGE_CCM}" +helm package charts/${MANAGER} --version ${CHART_VERSION} --destination ./ +helm package charts/${CCM} --version ${CHART_VERSION} --destination ./ + +echodate "Publishing helm charts to OCI registry ${REGISTRY_HOST}/${REPOSITORY_PREFIX}" +helm push ${CHART_PACKAGE_MANAGER} oci://${REGISTRY_HOST}/${REPOSITORY_PREFIX} +helm push ${CHART_PACKAGE_CCM} oci://${REGISTRY_HOST}/${REPOSITORY_PREFIX} + +rm ${CHART_PACKAGE_MANAGER} +rm ${CHART_PACKAGE_CCM} +helm registry logout ${REGISTRY_HOST} diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh index 1191506..abc9e01 100755 --- a/hack/verify-boilerplate.sh +++ b/hack/verify-boilerplate.sh @@ -21,4 +21,5 @@ cd $(dirname $0)/.. boilerplate \ -boilerplates hack/boilerplate/ \ -exclude config \ + -exclude charts \ -exclude Makefile \ No newline at end of file