From c480d2bd371ce9c305e71889ae67224512e1dcbc Mon Sep 17 00:00:00 2001 From: seifrajhi Date: Mon, 3 Jun 2024 10:20:54 +0200 Subject: [PATCH] Multi-cluster, Multi-tenant Kustomize Example --- .../README.md | 21 ++++++++++++++++ .../bases/app/deployment.yaml | 24 +++++++++++++++++++ .../bases/app/kustomization.yaml | 9 +++++++ .../bases/app/service.yaml | 18 ++++++++++++++ .../overlays/clusters/eu/kustomization.yaml | 6 +++++ .../overlays/clusters/us/kustomization.yaml | 6 +++++ .../overlays/plans/paid/kustomization.yaml | 5 ++++ .../overlays/plans/trial/kustomization.yaml | 8 +++++++ .../trial/patch-deployment-replicas.yaml | 6 +++++ .../tenant-envs/one/prod/kustomization.yaml | 11 +++++++++ .../tenant-envs/one/prod/namespace.yaml | 5 ++++ .../tenant-envs/one/test/kustomization.yaml | 11 +++++++++ .../tenant-envs/one/test/namespace.yaml | 5 ++++ .../tenant-envs/two/prod/kustomization.yaml | 11 +++++++++ .../tenant-envs/two/prod/namespace.yaml | 5 ++++ .../tenant-envs/two/test/kustomization.yaml | 11 +++++++++ .../tenant-envs/two/test/namespace.yaml | 5 ++++ 17 files changed, 167 insertions(+) create mode 100644 multi-cluster-multi-tenant-kustomize/README.md create mode 100644 multi-cluster-multi-tenant-kustomize/bases/app/deployment.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/bases/app/kustomization.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/bases/app/service.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/clusters/eu/kustomization.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/clusters/us/kustomization.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/plans/paid/kustomization.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/plans/trial/kustomization.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/plans/trial/patch-deployment-replicas.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/prod/kustomization.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/prod/namespace.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/test/kustomization.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/test/namespace.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/prod/kustomization.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/prod/namespace.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/test/kustomization.yaml create mode 100644 multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/test/namespace.yaml diff --git a/multi-cluster-multi-tenant-kustomize/README.md b/multi-cluster-multi-tenant-kustomize/README.md new file mode 100644 index 0000000..0359e4e --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/README.md @@ -0,0 +1,21 @@ +# Multi-cluster, Multi-tenant Kustomize Example + +This repository shows an example of how to use Kustomize's bases and overlays to maintain manifests for an application that requires one instance of the application to be deployed per tenant and per environment. + +Bases are configurations that inherit nothing. Overlays are configurations that inherit from somewhere. Overlays can inherit from bases or from other overlays. + +Our example has just one base, the example app represented by a single nginx deployment. + +In overlays, we have `clusters`, `plans` and `tenant-envs`. + + 1. `clusters`: We have one directory per region. If a tenant-env should be in the us, you add it as a base to the `us/kustomization.yaml`. If a tenant-env should be in the eu, you add it to the `eu/kustomization.yaml` bases. + + 1. `plans`: The plans overlay is where you'd put configuration that is different per plan. In our example trial tenants get less replicas then paying tenants. + + 1. `tenant-envs`: Our example has a `test` and a `prod` environment per client. Both tenant environments go onto the same cluster. The tenant-env overlays are where you put configuration that is specific to an env. E.g. the database connection should be unique per tenant per env. The tenant envs would also be a good place to give a certain tenant a specific version of the app (e.g. a hotfix) by overwriting the image tags for that tenant and possibly in the tenants test env first. + +Adopting a repository structure like this to manage multiple tenants makes it intuitive to understand where certain changes should be made while at the same time reducing the amount of duplicate manifests to a minimum. + +Applying a configuration to a cluster ist just one `kustomize build overlays/clusters/eu | kubectl apply -f -` command. + +Kustomize has recently been included into kubectl. Once that's released a simple `kubectl apply -f overlays/clusters/eu` is good enough. diff --git a/multi-cluster-multi-tenant-kustomize/bases/app/deployment.yaml b/multi-cluster-multi-tenant-kustomize/bases/app/deployment.yaml new file mode 100644 index 0000000..8ce1e14 --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/bases/app/deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app: app + name: app +spec: + replicas: 2 + selector: + matchLabels: + app: app + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + app: app + spec: + containers: + - image: nginx + name: nginx + resources: {} +status: {} diff --git a/multi-cluster-multi-tenant-kustomize/bases/app/kustomization.yaml b/multi-cluster-multi-tenant-kustomize/bases/app/kustomization.yaml new file mode 100644 index 0000000..0ade6df --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/bases/app/kustomization.yaml @@ -0,0 +1,9 @@ +commonLabels: + app.kubernetes.io/name: app + app.kubernetes.io/component: frontend + app.kubernetes.io/part-of: app + app.kubernetes.io/managed-by: kustomize + +resources: +- deployment.yaml +- service.yaml diff --git a/multi-cluster-multi-tenant-kustomize/bases/app/service.yaml b/multi-cluster-multi-tenant-kustomize/bases/app/service.yaml new file mode 100644 index 0000000..619547d --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/bases/app/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + app: app + name: app +spec: + ports: + - name: 80-8080 + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: app + type: ClusterIP +status: + loadBalancer: {} diff --git a/multi-cluster-multi-tenant-kustomize/overlays/clusters/eu/kustomization.yaml b/multi-cluster-multi-tenant-kustomize/overlays/clusters/eu/kustomization.yaml new file mode 100644 index 0000000..ac34620 --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/clusters/eu/kustomization.yaml @@ -0,0 +1,6 @@ +bases: +- ../../tenant-envs/one/prod +- ../../tenant-envs/one/test + +commonLabels: + example.com/region: eu diff --git a/multi-cluster-multi-tenant-kustomize/overlays/clusters/us/kustomization.yaml b/multi-cluster-multi-tenant-kustomize/overlays/clusters/us/kustomization.yaml new file mode 100644 index 0000000..3e14ff6 --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/clusters/us/kustomization.yaml @@ -0,0 +1,6 @@ +bases: +- ../../tenant-envs/two/prod +- ../../tenant-envs/two/test + +commonLabels: + example.com/region: us diff --git a/multi-cluster-multi-tenant-kustomize/overlays/plans/paid/kustomization.yaml b/multi-cluster-multi-tenant-kustomize/overlays/plans/paid/kustomization.yaml new file mode 100644 index 0000000..225abd4 --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/plans/paid/kustomization.yaml @@ -0,0 +1,5 @@ +bases: +- ../../../bases/app + +commonLabels: + example.com/plan: paid diff --git a/multi-cluster-multi-tenant-kustomize/overlays/plans/trial/kustomization.yaml b/multi-cluster-multi-tenant-kustomize/overlays/plans/trial/kustomization.yaml new file mode 100644 index 0000000..e0acc79 --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/plans/trial/kustomization.yaml @@ -0,0 +1,8 @@ +bases: +- ../../../bases/app + +commonLabels: + example.com/plan: trial + +patches: +- patch-deployment-replicas.yaml diff --git a/multi-cluster-multi-tenant-kustomize/overlays/plans/trial/patch-deployment-replicas.yaml b/multi-cluster-multi-tenant-kustomize/overlays/plans/trial/patch-deployment-replicas.yaml new file mode 100644 index 0000000..08a567d --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/plans/trial/patch-deployment-replicas.yaml @@ -0,0 +1,6 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app +spec: + replicas: 1 diff --git a/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/prod/kustomization.yaml b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/prod/kustomization.yaml new file mode 100644 index 0000000..0f3993c --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/prod/kustomization.yaml @@ -0,0 +1,11 @@ +namespace: one-prod + +bases: +- ../../../plans/paid + +commonLabels: + app.kubernetes.io/instance: tenant-one + example.com/stage: prod + +resources: +- namespace.yaml diff --git a/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/prod/namespace.yaml b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/prod/namespace.yaml new file mode 100644 index 0000000..a811a57 --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/prod/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: one-prod diff --git a/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/test/kustomization.yaml b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/test/kustomization.yaml new file mode 100644 index 0000000..848a0ff --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/test/kustomization.yaml @@ -0,0 +1,11 @@ +namespace: one-test + +bases: +- ../../../plans/paid + +commonLabels: + app.kubernetes.io/instance: tenant-one + example.com/stage: test + +resources: +- namespace.yaml diff --git a/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/test/namespace.yaml b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/test/namespace.yaml new file mode 100644 index 0000000..7ed5474 --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/one/test/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: one-test diff --git a/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/prod/kustomization.yaml b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/prod/kustomization.yaml new file mode 100644 index 0000000..cf3dfea --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/prod/kustomization.yaml @@ -0,0 +1,11 @@ +namespace: two-prod + +bases: +- ../../../plans/trial + +commonLabels: + app.kubernetes.io/instance: tenant-two + example.com/stage: prod + +resources: +- namespace.yaml diff --git a/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/prod/namespace.yaml b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/prod/namespace.yaml new file mode 100644 index 0000000..b9dc769 --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/prod/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: two-prod diff --git a/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/test/kustomization.yaml b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/test/kustomization.yaml new file mode 100644 index 0000000..70b60e5 --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/test/kustomization.yaml @@ -0,0 +1,11 @@ +namespace: two-test + +bases: +- ../../../plans/trial + +commonLabels: + app.kubernetes.io/instance: tenant-two + example.com/stage: test + +resources: +- namespace.yaml diff --git a/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/test/namespace.yaml b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/test/namespace.yaml new file mode 100644 index 0000000..e025b4f --- /dev/null +++ b/multi-cluster-multi-tenant-kustomize/overlays/tenant-envs/two/test/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: two-test