Updates node_pool to use kubeconfig on control plane node to delete n…

README.md

@@ -67,6 +67,8 @@ terraform apply -target=module.node_pool_green
 
 At which point, you can either destroy the old pool, or taint/evict pods, etc. once this new pool connects.
 
+When destroying node pools, you must have `KUBECONFIG` set to the cluster's kubeconfig on your local `kubectl` instance in order for the cluster to [cordon, drain](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/), and delete the node from the cluster (otherwise, the node remains in a `NotReady` state, once the machine itself has been terminated).
+
 ## GPU Node Pools
 
 The `gpu_node_pool` module provisions and configures GPU nodes for use with your Kubernetes cluster. The module definition requires `count_gpu` (defaults to "0"), and `plan_gpu` (defaults to `g2.large`). See [`examples/gpu_node_pool.tf`](examples/gpu_node_pool.tf) for usage.

modules/gpu_node_pool/main.tf

@@ -20,5 +20,20 @@ resource "metal_device" "gpu_node" {
 
   billing_cycle = "hourly"
   project_id    = var.project_id
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl cordon ${self.hostname} || echo \"If unsuccessful, set KUBECONFIG for your local kubectl for cluster to active, and cordon ${self.hostname} manually.\""
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl drain ${self.hostname} --delete-local-data --ignore-daemonsets || echo \"If unsuccessful, set KUBECONFIG for your local kubectl for cluster to active, and drain ${self.hostname} manually.\""
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl delete node ${self.hostname} || echo \"If unsuccessful, set KUBECONFIG for your local kubectl for cluster to active, and delete node ${self.hostname} manually.\""
+  }
 }
 

modules/node_pool/main.tf

@@ -20,6 +20,21 @@ resource "metal_device" "x86_node" {
 
   billing_cycle = "hourly"
   project_id    = var.project_id
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl cordon ${self.hostname} || echo \"If unsuccessful, set KUBECONFIG for your local kubectl for cluster to active, and cordon ${self.hostname} manually.\""
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl drain ${self.hostname} --delete-local-data --ignore-daemonsets || echo \"If unsuccessful, set KUBECONFIG for your local kubectl for cluster to active, and drain ${self.hostname} manually.\""
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl delete node ${self.hostname} || echo \"If unsuccessful, set KUBECONFIG for your local kubectl for cluster to active, and delete node ${self.hostname} manually.\""
+  }
 }
 
 resource "metal_device" "arm_node" {
@@ -33,4 +48,19 @@ resource "metal_device" "arm_node" {
 
   billing_cycle = "hourly"
   project_id    = var.project_id
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl cordon ${self.hostname} || echo \"If unsuccessful, set KUBECONFIG for your local kubectl for cluster to active, and cordon ${self.hostname} manually.\""
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl drain ${self.hostname} --delete-local-data --ignore-daemonsets || echo \"If unsuccessful, set KUBECONFIG for your local kubectl for cluster to active, and drain ${self.hostname} manually.\""
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "kubectl delete node ${self.hostname} || echo \"If unsuccessful, set KUBECONFIG for your local kubectl for cluster to active, and delete node ${self.hostname} manually.\""
+  }
 }

modules/node_pool/variables.tf

@@ -58,3 +58,4 @@ variable "storage" {
   type        = string
   description = "Configure Storage ('ceph' or 'openebs') Operator"
 }
+