Setup two separate paths for development and production environments …

…(across provisioning, configuration, and deployment)

README.md

@@ -70,34 +70,44 @@ gcloud iam service-accounts keys create isidro-provisioner.json \
     --iam-account="isidro-provisioner@$GOOGLE_PROJECT.iam.gserviceaccount.com"
 ```
 
-Navigate to the [provisioning/](provisioning/) directory, then set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable, with something like:
+Navigate to the [development](provisioning/dev/) or [production](provisioning/prod/) directory, then set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable, with something like:
 ```bash
 export GOOGLE_APPLICATION_CREDENTIALS=../isidro-provisioner.json
 ```
 
-Setup secondary IP ranges in the desired regions and subnets, then [run Terraform provisioning, with variable changes/overrides where required](provisioning/).  Something like:
+Run Terraform provisioning, with something like:
 ```bash
 terraform init
 terraform apply
 ```
 
-Create kubecontext configurations for the three provisioned clusters:
+Create kubecontext configurations for the provisioned clusters:
 ```bash
+# For development and production setups
 gcloud container clusters get-credentials isidro-us --region us-central1
-gcloud container clusters get-credentials isidro-fi --region europe-north1
-gcloud container clusters get-credentials isidro-br --region southamerica-east1
 gcloud container clusters get-credentials isidro-config --region northamerica-northeast1
 kubectl config rename-context gke_"$GOOGLE_PROJECT"_us-central1_isidro-us isidro-us
+kubectl config rename-context gke_"$GOOGLE_PROJECT"_northamerica-northeast1_isidro-config isidro-config
+```
+
+```bash
+# For production setups also run
+gcloud container clusters get-credentials isidro-fi --region europe-north1
+gcloud container clusters get-credentials isidro-br --region southamerica-east1
 kubectl config rename-context gke_"$GOOGLE_PROJECT"_europe-north1_isidro-fi isidro-fi
 kubectl config rename-context gke_"$GOOGLE_PROJECT"_southamerica-east1_isidro-br isidro-br
-kubectl config rename-context gke_"$GOOGLE_PROJECT"_northamerica-northeast1_isidro-config isidro-config
 ```
 
 ### Enable GMP
 
 Enable Managed Prometheus for the US and Europe clusters:
 ```bash
+# For development and production setups
 gcloud beta container clusters update isidro-us --region us-central1 --enable-managed-prometheus
+```
+
+```bash
+# For production setups also run
 gcloud beta container clusters update isidro-fi --region europe-north1 --enable-managed-prometheus
 gcloud beta container clusters update isidro-br --region southamerica-east1 --enable-managed-prometheus
 ```
@@ -120,10 +130,16 @@ Add helm repositories:
 helm repo add mattermost https://helm.mattermost.com
 ```
 
-Setup skaffold files and credentials:
+Setup skaffold credentials:
 ```bash
 export GOOGLE_APPLICATION_CREDENTIALS=isidro-skaffold.json
+```
+
+### Development environments
 
+Hydrate configurations:
+```bash
+# For development
 cp skaffold.dev.yaml skaffold.yaml
 sed -i "s/GOOGLE_PROJECT/$GOOGLE_PROJECT/g" skaffold.yaml
 sed -i "s/MATTERMOST_DOMAIN/$MATTERMOST_DOMAIN/g" skaffold.yaml
@@ -134,14 +150,25 @@ cp vendor/configconnector-setup.dev.yaml vendor/configconnector-setup.yaml
 sed -i "s/GOOGLE_PROJECT/$GOOGLE_PROJECT/g" vendor/configconnector-setup.yaml
 ```
 
-### Development environments
-
 Make any required `skaffold.yaml` configuration changes, then run skaffold:
 ```bash
 skaffold dev
 ```
 
-### Persistent environments
+### Production environments
+
+Hydrate confiigurations:
+```bash
+# For production
+cp skaffold.prod.yaml skaffold.yaml
+sed -i "s/GOOGLE_PROJECT/$GOOGLE_PROJECT/g" skaffold.yaml
+sed -i "s/MATTERMOST_DOMAIN/$MATTERMOST_DOMAIN/g" skaffold.yaml
+sed -i "s/ISIDRO_DOMAIN/$ISIDRO_DOMAIN/g" skaffold.yaml
+sed -i "s/DNS_ZONE_NAME/$DNS_ZONE_NAME/g" skaffold.yaml
+
+cp vendor/configconnector-setup.dev.yaml vendor/configconnector-setup.yaml
+sed -i "s/GOOGLE_PROJECT/$GOOGLE_PROJECT/g" vendor/configconnector-setup.yaml
+```
 
 Make any required `skaffold.yaml` configuration changes, then run skaffold:
 ```bash
@@ -200,7 +227,7 @@ curl -X POST https://isidro.example.com/api/v1/submit \
 
 ## Deprovisioning
 
-In the [Terraform provisioning directory](provisioning/).  Run:
+In the Terraform provisioning directory, run:
 ```bash
 terraform destroy
 ```

provisioning/dev/main.tf

@@ -0,0 +1,73 @@
+data "google_project" "project" {}
+data "google_client_config" "default" {}
+
+module "foundations" {
+  source = "../modules/foundation"
+  project_number = data.google_project.project.number
+  project_id = data.google_project.project.project_id
+
+}
+
+module "primary" {
+  source                 = "../modules/instance"
+  name                   = "isidro-us"
+  vpc                    = module.foundations.vpc_name
+  auxiliary_range        = "172.16.0.0/18"
+  pods_range             = "172.16.64.0/19"
+  services_range         = "172.16.96.0/19"
+  region                 = "us-central1"
+  node_count             = 1
+  nodes_service_account  = module.foundations.nodes_sa_email
+  spot                   = false
+  machine_type           = "n2d-standard-2"
+  binauthz_attestor_name = module.foundations.binauthz_attestor
+  providers = {
+    kubernetes = kubernetes.primary
+  }
+}
+
+provider "kubernetes" {
+  alias                  = "primary"
+  host                   = "https://${module.primary.endpoint}"
+  token                  = data.google_client_config.default.access_token
+  cluster_ca_certificate = base64decode(module.primary.ca_certificate)
+}
+
+module "config" {
+  source                 = "../modules/instance"
+  name                   = "isidro-config"
+  vpc                    = module.foundations.vpc_name
+  auxiliary_range        = "172.17.0.0/18"
+  pods_range             = "172.17.64.0/19"
+  services_range         = "172.17.96.0/19"
+  region                 = "northamerica-northeast1"
+  node_count             = 1
+  nodes_service_account  = module.foundations.nodes_sa_email
+  spot                   = false
+  machine_type           = "n2d-standard-2"
+  binauthz_attestor_name = module.foundations.binauthz_attestor
+  providers = {
+    kubernetes = kubernetes.config
+  }
+}
+
+provider "kubernetes" {
+  alias                  = "config"
+  host                   = "https://${module.config.endpoint}"
+  token                  = data.google_client_config.default.access_token
+  cluster_ca_certificate = base64decode(module.config.ca_certificate)
+}
+
+resource "google_gke_hub_feature" "mci" {
+  depends_on = [
+    module.config
+  ]
+  name     = "multiclusteringress"
+  location = "global"
+  spec {
+    multiclusteringress {
+      config_membership = "projects/${data.google_project.project.project_id}/locations/global/memberships/${module.config.name}-membership"
+    }
+  }
+  provider = google-beta
+}

provisioning/dev/outputs.tf

@@ -0,0 +1,15 @@
+output "binauthz_keyring" {
+  value = module.foundations.binauthz_keyring
+}
+
+output "binauthz_keyring_location" {
+  value = module.foundations.binauthz_keyring_location
+}
+
+output "binauthz_attestor" {
+  value = module.foundations.binauthz_attestor
+}
+
+output "binauthz_key" {
+  value = module.foundations.binauthz_key
+}

provisioning/versions.tf → provisioning/dev/versions.tf

@@ -14,4 +14,7 @@ terraform {
       version = "~> 2.12.1"
     }
   }
-}
+}
+
+provider "google" {}
+provider "google-beta" {}

provisioning/iam-certbot.tf → provisioning/modules/foundation/iam-certbot.tf

@@ -4,14 +4,14 @@ resource "google_service_account" "certbot" {
 }
 
 resource "google_project_iam_member" "certbot_dns_admin" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/dns.admin"
   member  = "serviceAccount:${google_service_account.certbot.email}"
 }
 
 resource "google_project_iam_member" "certbot_workload_identity_user" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/iam.workloadIdentityUser"
-  member  = "serviceAccount:${data.google_project.project.project_id}.svc.id.goog[isidro/certbot]"
+  member  = "serviceAccount:${var.project_id}.svc.id.goog[isidro/certbot]"
 }
 

provisioning/iam-config-connector.tf → provisioning/modules/foundation/iam-config-connector.tf

@@ -4,14 +4,14 @@ resource "google_service_account" "config_connector" {
 }
 
 resource "google_project_iam_member" "config_connector_editor" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/editor"
   member  = "serviceAccount:${google_service_account.config_connector.email}"
 }
 
 resource "google_project_iam_member" "config_connector_workload_identity_user" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/iam.workloadIdentityUser"
-  member  = "serviceAccount:${data.google_project.project.project_id}.svc.id.goog[cnrm-system/cnrm-controller-manager]"
+  member  = "serviceAccount:${var.project_id}.svc.id.goog[cnrm-system/cnrm-controller-manager]"
 }
 

provisioning/modules/foundation/iam-mcs.tf

@@ -0,0 +1,5 @@
+resource "google_project_iam_member" "mcs_network_viewer" {
+  project = var.project_id
+  role    = "roles/compute.networkViewer"
+  member  = "serviceAccount:${var.project_id}.svc.id.goog[gke-mcs/gke-mcs-importer]"
+}

provisioning/iam-microservices.tf → provisioning/modules/foundation/iam-microservices.tf

@@ -4,15 +4,15 @@ resource "google_service_account" "tracing_microservices" {
 }
 
 resource "google_project_iam_member" "tracing_microservices_trace_writer" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/cloudtrace.agent"
   member  = "serviceAccount:${google_service_account.tracing_microservices.email}"
 }
 
 resource "google_project_iam_member" "tracing_microservices_workload_identity_user" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/iam.workloadIdentityUser"
-  member  = "serviceAccount:${data.google_project.project.project_id}.svc.id.goog[isidro/tracing-microservice]"
+  member  = "serviceAccount:${var.project_id}.svc.id.goog[isidro/tracing-microservice]"
 }
 
 resource "google_service_account" "db_microservices" {
@@ -21,21 +21,21 @@ resource "google_service_account" "db_microservices" {
 }
 
 resource "google_project_iam_member" "db_microservices_spanner_user" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/spanner.databaseUser"
   member  = "serviceAccount:${google_service_account.db_microservices.email}"
 }
 
 resource "google_project_iam_member" "db_microservices_trace_writer" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/cloudtrace.agent"
   member  = "serviceAccount:${google_service_account.db_microservices.email}"
 }
 
 resource "google_project_iam_member" "db_microservices_workload_identity_user" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/iam.workloadIdentityUser"
-  member  = "serviceAccount:${data.google_project.project.project_id}.svc.id.goog[isidro/db-client-microservice]"
+  member  = "serviceAccount:${var.project_id}.svc.id.goog[isidro/db-client-microservice]"
 }
 
 resource "google_service_account" "kubebash_microservices" {
@@ -44,13 +44,13 @@ resource "google_service_account" "kubebash_microservices" {
 }
 
 resource "google_project_iam_member" "kubebash_microservices_trace_writer" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/cloudtrace.agent"
   member  = "serviceAccount:${google_service_account.kubebash_microservices.email}"
 }
 
 resource "google_project_iam_member" "kubebash_microservices_workload_identity_user" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/iam.workloadIdentityUser"
-  member  = "serviceAccount:${data.google_project.project.project_id}.svc.id.goog[isidro/kubebash-microservice]"
+  member  = "serviceAccount:${var.project_id}.svc.id.goog[isidro/kubebash-microservice]"
 }

provisioning/iam-nodes.tf → provisioning/modules/foundation/iam-nodes.tf

@@ -4,37 +4,37 @@ resource "google_service_account" "nodes" {
 }
 
 resource "google_project_iam_member" "nodes_artifact_reader" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/artifactregistry.reader"
   member  = "serviceAccount:${google_service_account.nodes.email}"
 }
 
 resource "google_project_iam_member" "nodes_log_writer" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/logging.logWriter"
   member  = "serviceAccount:${google_service_account.nodes.email}"
 }
 
 resource "google_project_iam_member" "nodes_metric_writer" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/monitoring.metricWriter"
   member  = "serviceAccount:${google_service_account.nodes.email}"
 }
 
 resource "google_project_iam_member" "nodes_metric_reader" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/monitoring.viewer"
   member  = "serviceAccount:${google_service_account.nodes.email}"
 }
 
 resource "google_project_iam_member" "nodes_metadata_writer" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/stackdriver.resourceMetadata.writer"
   member  = "serviceAccount:${google_service_account.nodes.email}"
 }
 
 resource "google_project_iam_member" "nodes_object_reader" {
-  project = data.google_project.project.project_id
+  project = var.project_id
   role    = "roles/storage.objectViewer"
   member  = "serviceAccount:${google_service_account.nodes.email}"
 }

provisioning/monitoring.tf → provisioning/modules/foundation/monitoring.tf

@@ -1,5 +1,5 @@
 resource "google_monitoring_slo" "gatekeeper_latency" {
-  service = "canonical-ist:proj-${data.google_project.project.number}-default-gatekeeper"
+  service = "canonical-ist:proj-${var.project_number}-default-gatekeeper"
 
   slo_id       = "gatekeeper-latency"
   display_name = "Isidro SLO for latency"
@@ -15,7 +15,7 @@ resource "google_monitoring_slo" "gatekeeper_latency" {
 }
 
 resource "google_monitoring_slo" "gatekeeper_errors" {
-  service = "canonical-ist:proj-${data.google_project.project.number}-default-gatekeeper"
+  service = "canonical-ist:proj-${var.project_number}-default-gatekeeper"
 
   slo_id       = "gatekeeper-errors"
   display_name = "Isidro SLO for errors"

provisioning/outputs.tf → provisioning/modules/foundation/outputs.tf

@@ -12,4 +12,12 @@ output "binauthz_attestor" {
 
 output "binauthz_key" {
   value = google_kms_crypto_key.isidro.name
+}
+
+output "vpc_name" {
+  value = google_compute_network.isidro.name
+}
+
+output "nodes_sa_email" {
+  value = google_service_account.nodes.email
 }