From f6fc87190e081b3534044304ce11e225bf421f8f Mon Sep 17 00:00:00 2001
From: Frank <fvumbaca@users.noreply.github.com>
Date: Wed, 5 Jan 2022 10:46:46 -0500
Subject: [PATCH] Added options to specify custom storage on all nodes (#3)

* Added options to specify custom storage on all nodes

* Terraform fmt

* updated README
---
 README.md       |   8 +--
 master_nodes.tf | 112 ++++++++++++++++++-----------------
 outputs.tf      |   8 +--
 support_node.tf | 154 +++++++++++++++++++++++++-----------------------
 variables.tf    |  78 ++++++++++++------------
 versions.tf     |   4 +-
 worker_nodes.tf |  82 +++++++++++++-------------
 7 files changed, 226 insertions(+), 220 deletions(-)

diff --git a/README.md b/README.md
index e82f51d..ff0a554 100644
--- a/README.md
+++ b/README.md
@@ -26,7 +26,7 @@ A module for spinning up an expandable and flexible K3s server for your HomeLab.
 ```terraform
 module "k3s" {
   source  = "fvumbaca/k3s/proxmox"
-  version = "0.0.0"
+  version = ">= 0.0.0, < 1.0.0" # Get latest 0.X release
 
   authorized_keys_file = "authorized_keys"
 
@@ -82,12 +82,6 @@ output "kubeconfig" {
 }
 ```
 
-You may need to refresh your state:
-
-```sh
-terraform refresh
-```
-
 Finally output the config file:
 
 ```sh
diff --git a/master_nodes.tf b/master_nodes.tf
index 0914fff..b3272bc 100644
--- a/master_nodes.tf
+++ b/master_nodes.tf
@@ -1,22 +1,24 @@
 resource "macaddress" "k3s-masters" {
-    count = var.master_nodes_count
+  count = var.master_nodes_count
 }
 
 locals {
   master_node_settings = defaults(var.master_node_settings, {
-    cores = 2
-    sockets = 1
-    memory = 4096
-    disk_size = "20G"
-    user = "k3s"
+    cores        = 2
+    sockets      = 1
+    memory       = 4096
+    storage_type = "scsi"
+    storage_id   = "local-lvm"
+    disk_size    = "20G"
+    user         = "k3s"
   })
 
-  master_node_ips = [for i in range(var.master_nodes_count): cidrhost(var.control_plane_subnet, i+1)]
+  master_node_ips = [for i in range(var.master_nodes_count) : cidrhost(var.control_plane_subnet, i + 1)]
 }
 
 resource "random_password" "k3s-server-token" {
-  length = 32
-  special = false
+  length           = 32
+  special          = false
   override_special = "_%@"
 }
 
@@ -25,23 +27,23 @@ resource "proxmox_vm_qemu" "k3s-master" {
     proxmox_vm_qemu.k3s-support,
   ]
 
-  count = var.master_nodes_count
+  count       = var.master_nodes_count
   target_node = var.proxmox_node
-  name = "${var.cluster_name}-master-${count.index}"
+  name        = "${var.cluster_name}-master-${count.index}"
 
   clone = var.node_template
 
   pool = var.proxmox_resource_pool
 
   # cores = 2
-  cores = local.master_node_settings.cores
+  cores   = local.master_node_settings.cores
   sockets = local.master_node_settings.sockets
-  memory = local.master_node_settings.memory
+  memory  = local.master_node_settings.memory
 
   disk {
-    type = "scsi"
-    storage = "local-lvm"
-    size = local.master_node_settings.disk_size
+    type    = local.master_node_settings.storage_type
+    storage = local.master_node_settings.storage_id
+    size    = local.master_node_settings.disk_size
   }
 
   network {
@@ -56,51 +58,51 @@ resource "proxmox_vm_qemu" "k3s-master" {
   }
 
 
-  os_type   = "cloud-init"
+  os_type = "cloud-init"
 
   ciuser = local.master_node_settings.user
 
   ipconfig0 = "ip=${local.master_node_ips[count.index]}/${local.lan_subnet_cidr_bitnum},gw=${var.network_gateway}"
 
-    sshkeys = file(var.authorized_keys_file)
-
-    connection {
-      type     = "ssh"
-      user     = local.master_node_settings.user
-      host     = local.master_node_ips[count.index]
-    }
-
-    provisioner "remote-exec" {
-      inline = [
-        templatefile("${path.module}/scripts/install-k3s-server.sh.tftpl", {
-          mode = "server"
-          tokens = [random_password.k3s-server-token.result]
-          alt_names = concat([local.support_node_ip], var.api_hostnames)
-          server_hosts = []
-          node_taints = ["CriticalAddonsOnly=true:NoExecute"]
-          disable = var.k3s_disable_components
-          datastores = [{
-            host = "${local.support_node_ip}:3306"
-            name = "k3s"
-            user = "k3s"
-            password = random_password.k3s-master-db-password.result
-          }]
-        })
-      ]
-    }
-  }
+  sshkeys = file(var.authorized_keys_file)
 
-  data "external" "kubeconfig" {
-    depends_on = [
-      proxmox_vm_qemu.k3s-support,
-      proxmox_vm_qemu.k3s-master
-    ]
+  connection {
+    type = "ssh"
+    user = local.master_node_settings.user
+    host = local.master_node_ips[count.index]
+  }
 
-    program = [
-      "/usr/bin/ssh",
-      "-o UserKnownHostsFile=/dev/null",
-      "-o StrictHostKeyChecking=no",
-      "${local.master_node_settings.user}@${local.master_node_ips[0]}",
-      "echo '{\"kubeconfig\":\"'$(sudo cat /etc/rancher/k3s/k3s.yaml | base64)'\"}'"
+  provisioner "remote-exec" {
+    inline = [
+      templatefile("${path.module}/scripts/install-k3s-server.sh.tftpl", {
+        mode         = "server"
+        tokens       = [random_password.k3s-server-token.result]
+        alt_names    = concat([local.support_node_ip], var.api_hostnames)
+        server_hosts = []
+        node_taints  = ["CriticalAddonsOnly=true:NoExecute"]
+        disable      = var.k3s_disable_components
+        datastores = [{
+          host     = "${local.support_node_ip}:3306"
+          name     = "k3s"
+          user     = "k3s"
+          password = random_password.k3s-master-db-password.result
+        }]
+      })
     ]
   }
+}
+
+data "external" "kubeconfig" {
+  depends_on = [
+    proxmox_vm_qemu.k3s-support,
+    proxmox_vm_qemu.k3s-master
+  ]
+
+  program = [
+    "/usr/bin/ssh",
+    "-o UserKnownHostsFile=/dev/null",
+    "-o StrictHostKeyChecking=no",
+    "${local.master_node_settings.user}@${local.master_node_ips[0]}",
+    "echo '{\"kubeconfig\":\"'$(sudo cat /etc/rancher/k3s/k3s.yaml | base64)'\"}'"
+  ]
+}
diff --git a/outputs.tf b/outputs.tf
index 7b40fd9..3f08dec 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -1,6 +1,6 @@
 
 output "k3s_db_password" {
-  value = random_password.k3s-master-db-password.result
+  value     = random_password.k3s-master-db-password.result
   sensitive = true
 }
 
@@ -17,7 +17,7 @@ output "k3s_db_host" {
 }
 
 output "root_db_password" {
-  value = random_password.support-db-password.result
+  value     = random_password.support-db-password.result
   sensitive = true
 }
 
@@ -34,7 +34,7 @@ output "master_node_ips" {
 }
 
 output "k3s_server_token" {
-  value = random_password.k3s-server-token.result
+  value     = random_password.k3s-server-token.result
   sensitive = true
 }
 
@@ -43,7 +43,7 @@ output "k3s_master_node_ips" {
 }
 
 output "k3s_kubeconfig" {
-  value = replace(base64decode(replace(data.external.kubeconfig.result.kubeconfig, " ", "")), "server: https://127.0.0.1:6443", "server: https://${local.support_node_ip}:6443")
+  value     = replace(base64decode(replace(data.external.kubeconfig.result.kubeconfig, " ", "")), "server: https://127.0.0.1:6443", "server: https://${local.support_node_ip}:6443")
   sensitive = true
 }
 
diff --git a/support_node.tf b/support_node.tf
index 6b1c5cf..9334419 100644
--- a/support_node.tf
+++ b/support_node.tf
@@ -3,11 +3,15 @@ resource "macaddress" "k3s-support" {}
 
 locals {
   support_node_settings = defaults(var.support_node_settings, {
-    cores = 2
+    cores   = 2
     sockets = 1
-    memory = 4096
-    disk_size = "10G"
-    user = "support"
+    memory  = 4096
+
+
+    storage_type = "scsi"
+    storage_id   = "local-lvm"
+    disk_size    = "10G"
+    user         = "support"
 
     db_name = "k3s"
     db_user = "k3s"
@@ -21,79 +25,79 @@ locals {
 }
 
 resource "proxmox_vm_qemu" "k3s-support" {
-    target_node = var.proxmox_node
-    name = join("-", [var.cluster_name, "support"])
-
-    clone = var.node_template
-
-    pool = var.proxmox_resource_pool
-
-    # cores = 2
-    cores = local.support_node_settings.cores
-    sockets = local.support_node_settings.sockets
-    memory = local.support_node_settings.memory
-
-    disk {
-      type = "scsi"
-      storage = "local-lvm"
-      size = local.support_node_settings.disk_size
-    }
-
-    network {
-      bridge    = "vmbr0"
-      firewall  = true
-      link_down = false
-      macaddr   = upper(macaddress.k3s-support.address)
-      model     = "virtio"
-      queues    = 0
-      rate      = 0
-      tag       = -1
-    }
-
-
-    os_type   = "cloud-init"
-
-    ciuser = local.support_node_settings.user
-
-    ipconfig0 = "ip=${local.support_node_ip}/${local.lan_subnet_cidr_bitnum},gw=${var.network_gateway}"
-
-    sshkeys = file(var.authorized_keys_file)
-
-    connection {
-      type     = "ssh"
-      user     = local.support_node_settings.user
-      host     = local.support_node_ip
-    }
-
-    provisioner "file" {
-      destination = "/tmp/install.sh"
-      content = templatefile("${path.module}/scripts/install-support-apps.sh.tftpl", {
-        root_password = random_password.support-db-password.result
-
-        k3s_database = local.support_node_settings.db_name
-        k3s_user = local.support_node_settings.db_user
-        k3s_password = random_password.k3s-master-db-password.result
-      })
-    }
-
-    provisioner "remote-exec" {
-      inline = [
-        "chmod u+x /tmp/install.sh",
-        "/tmp/install.sh",
-        "rm -r /tmp/install.sh",
-      ]
-    }
+  target_node = var.proxmox_node
+  name        = join("-", [var.cluster_name, "support"])
+
+  clone = var.node_template
+
+  pool = var.proxmox_resource_pool
+
+  # cores = 2
+  cores   = local.support_node_settings.cores
+  sockets = local.support_node_settings.sockets
+  memory  = local.support_node_settings.memory
+
+  disk {
+    type    = local.support_node_settings.storage_type
+    storage = local.support_node_settings.storage_id
+    size    = local.support_node_settings.disk_size
+  }
+
+  network {
+    bridge    = "vmbr0"
+    firewall  = true
+    link_down = false
+    macaddr   = upper(macaddress.k3s-support.address)
+    model     = "virtio"
+    queues    = 0
+    rate      = 0
+    tag       = -1
+  }
+
+
+  os_type = "cloud-init"
+
+  ciuser = local.support_node_settings.user
+
+  ipconfig0 = "ip=${local.support_node_ip}/${local.lan_subnet_cidr_bitnum},gw=${var.network_gateway}"
+
+  sshkeys = file(var.authorized_keys_file)
+
+  connection {
+    type = "ssh"
+    user = local.support_node_settings.user
+    host = local.support_node_ip
+  }
+
+  provisioner "file" {
+    destination = "/tmp/install.sh"
+    content = templatefile("${path.module}/scripts/install-support-apps.sh.tftpl", {
+      root_password = random_password.support-db-password.result
+
+      k3s_database = local.support_node_settings.db_name
+      k3s_user     = local.support_node_settings.db_user
+      k3s_password = random_password.k3s-master-db-password.result
+    })
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "chmod u+x /tmp/install.sh",
+      "/tmp/install.sh",
+      "rm -r /tmp/install.sh",
+    ]
+  }
 }
 
 resource "random_password" "support-db-password" {
-  length = 16
-  special = false
+  length           = 16
+  special          = false
   override_special = "_%@"
 }
 
 resource "random_password" "k3s-master-db-password" {
-  length = 16
-  special = false
+  length           = 16
+  special          = false
   override_special = "_%@"
 }
 
@@ -108,19 +112,19 @@ resource "null_resource" "k3s_nginx_config" {
   }
 
   connection {
-    type     = "ssh"
-    user     = local.support_node_settings.user
-    host     = local.support_node_ip
+    type = "ssh"
+    user = local.support_node_settings.user
+    host = local.support_node_ip
   }
 
   provisioner "file" {
     destination = "/tmp/nginx.conf"
     content = templatefile("${path.module}/config/nginx.conf.tftpl", {
-      k3s_server_hosts = [ for ip in local.master_node_ips:
+      k3s_server_hosts = [for ip in local.master_node_ips :
         "${ip}:6443"
       ]
       k3s_nodes = concat(local.master_node_ips, [
-        for node in local.listed_worker_nodes:
+        for node in local.listed_worker_nodes :
         node.ip
       ])
     })
diff --git a/variables.tf b/variables.tf
index 1ea3c75..86892af 100644
--- a/variables.tf
+++ b/variables.tf
@@ -1,16 +1,16 @@
 variable "proxmox_node" {
   description = "Proxmox node to create VMs on."
-  type = string
+  type        = string
 }
 
 variable "authorized_keys_file" {
   description = "Path to file containing public SSH keys for remoting into nodes."
-  type = string
+  type        = string
 }
 
 variable "network_gateway" {
   description = "IP address of the network gateway."
-  type = string
+  type        = string
   validation {
     # condition     = can(regex("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}/[0-9]{1,2}$", var.network_gateway))
     condition     = can(regex("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", var.network_gateway))
@@ -23,7 +23,7 @@ variable "lan_subnet" {
 Subnet used by the LAN network. Note that only the bit count number at the end
 is acutally used, and all other subnets provided are secondary subnets.
 EOF
-  type = string
+  type        = string
   validation {
     condition     = can(regex("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}/[0-9]{1,2}$", var.lan_subnet))
     error_message = "The lan_subnet value must be a valid cidr range."
@@ -33,7 +33,7 @@ EOF
 variable "control_plane_subnet" {
   description = <<EOF
 EOF
-  type = string
+  type        = string
   validation {
     condition     = can(regex("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}/[0-9]{1,2}$", var.control_plane_subnet))
     error_message = "The control_plane_subnet value must be a valid cidr range."
@@ -41,13 +41,13 @@ EOF
 }
 
 variable "cluster_name" {
-  default = "k3s"
-  type = string
+  default     = "k3s"
+  type        = string
   description = "Name of the cluster used for prefixing cluster components (ie nodes)."
 }
 
 variable "node_template" {
-  type = string
+  type        = string
   description = <<EOF
 Proxmox vm to use as a base template for all nodes. Can be a template or
 another vm that supports cloud-init.
@@ -56,67 +56,71 @@ EOF
 
 variable "proxmox_resource_pool" {
   description = "Resource pool name to use in proxmox to better organize nodes."
-  type = string
-  default = ""
+  type        = string
+  default     = ""
 }
 
 variable "support_node_settings" {
   type = object({
-    cores = optional(number),
-    sockets = optional(number),
-    memory = optional(number),
-    disk_size = optional(string),
-    user = optional(string),
-
-    db_name = optional(string),
-    db_user = optional(string),
+    cores        = optional(number),
+    sockets      = optional(number),
+    memory       = optional(number),
+    storage_type = optional(string),
+    storage_id   = optional(string),
+    disk_size    = optional(string),
+    user         = optional(string),
+    db_name      = optional(string),
+    db_user      = optional(string),
   })
 }
 
 variable "master_nodes_count" {
   description = "Number of master nodes."
-  default = 2
-  type = number
+  default     = 2
+  type        = number
 }
 
 variable "master_node_settings" {
   type = object({
-    cores = optional(number),
-    sockets = optional(number),
-    memory = optional(number),
-    disk_size = optional(string),
-    user = optional(string),
+    cores        = optional(number),
+    sockets      = optional(number),
+    memory       = optional(number),
+    storage_type = optional(string),
+    storage_id   = optional(string),
+    disk_size    = optional(string),
+    user         = optional(string),
   })
 }
 
-
 variable "node_pools" {
   description = "Node pool definitions for the cluster."
   type = list(object({
 
-    name = string,
-    size = number,
+    name   = string,
+    size   = number,
     subnet = string,
 
     taints = optional(list(string)),
 
-    cores = optional(number),
-    sockets = optional(number),
-    memory = optional(number),
-    disk_size = optional(string),
-    user = optional(string),
+    cores        = optional(number),
+    sockets      = optional(number),
+    memory       = optional(number),
+    storage_type = optional(string),
+    storage_id   = optional(string),
+    disk_size    = optional(string),
+    user         = optional(string),
 
     template = optional(string),
   }))
 }
 variable "api_hostnames" {
   description = "Alternative hostnames for the API server."
-  type = list(string)
-  default = []
+  type        = list(string)
+  default     = []
 }
 
 variable "k3s_disable_components" {
   description = "List of components to disable. Ref: https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/#kubernetes-components"
-  type = list(string)
-  default = []
+  type        = list(string)
+  default     = []
 }
diff --git a/versions.tf b/versions.tf
index 8e9cb0a..5af36ae 100644
--- a/versions.tf
+++ b/versions.tf
@@ -1,12 +1,12 @@
 terraform {
   required_providers {
     proxmox = {
-      source = "Telmate/proxmox"
+      source  = "Telmate/proxmox"
       version = "2.9.3"
     }
 
     macaddress = {
-      source = "ivoronin/macaddress"
+      source  = "ivoronin/macaddress"
       version = "0.3.0"
     }
   }
diff --git a/worker_nodes.tf b/worker_nodes.tf
index 1118de1..0ea55ef 100644
--- a/worker_nodes.tf
+++ b/worker_nodes.tf
@@ -1,29 +1,31 @@
 resource "macaddress" "k3s-workers" {
-    for_each = local.mapped_worker_nodes
+  for_each = local.mapped_worker_nodes
 }
 
 locals {
 
   listed_worker_nodes = flatten([
-    for pool in var.node_pools:
+    for pool in var.node_pools :
     [
-      for i in range(pool.size):
+      for i in range(pool.size) :
       merge(defaults(pool, {
-        cores = 2
-        sockets = 1
-        memory = 4096
-        disk_size = "20G"
-        user = "k3s"
-        template = var.node_template
-      }), {
-        i = i
+        cores        = 2
+        sockets      = 1
+        memory       = 4096
+        storage_type = "scsi"
+        storage_id   = "local-lvm"
+        disk_size    = "20G"
+        user         = "k3s"
+        template     = var.node_template
+        }), {
+        i  = i
         ip = cidrhost(pool.subnet, i)
       })
     ]
   ])
 
   mapped_worker_nodes = {
-    for node in local.listed_worker_nodes: "${node.name}-${node.i}" => node
+    for node in local.listed_worker_nodes : "${node.name}-${node.i}" => node
   }
 
 }
@@ -37,21 +39,21 @@ resource "proxmox_vm_qemu" "k3s-worker" {
   for_each = local.mapped_worker_nodes
 
   target_node = var.proxmox_node
-  name = "${var.cluster_name}-${each.key}"
+  name        = "${var.cluster_name}-${each.key}"
 
   clone = each.value.template
 
   pool = var.proxmox_resource_pool
 
   # cores = 2
-  cores = each.value.cores
+  cores   = each.value.cores
   sockets = each.value.sockets
-  memory = each.value.memory
+  memory  = each.value.memory
 
   disk {
-    type = "scsi"
-    storage = "local-lvm"
-    size = each.value.disk_size
+    type    = each.value.storage_type
+    storage = each.value.storage_id
+    size    = each.value.disk_size
   }
 
   network {
@@ -65,33 +67,33 @@ resource "proxmox_vm_qemu" "k3s-worker" {
     tag       = -1
   }
 
-  os_type   = "cloud-init"
+  os_type = "cloud-init"
 
   ciuser = each.value.user
 
   ipconfig0 = "ip=${each.value.ip}/${local.lan_subnet_cidr_bitnum},gw=${var.network_gateway}"
 
-    sshkeys = file(var.authorized_keys_file)
-
-    connection {
-      type     = "ssh"
-      user     = each.value.user
-      host     = each.value.ip
-    }
-
-    provisioner "remote-exec" {
-      inline = [
-        templatefile("${path.module}/scripts/install-k3s-server.sh.tftpl", {
-          mode = "agent"
-          tokens = [random_password.k3s-server-token.result]
-          alt_names = []
-          disable = []
-          server_hosts = ["https://${local.support_node_ip}:6443"]
-          node_taints = each.value.taints
-          datastores = []
-        })
-      ]
-    }
+  sshkeys = file(var.authorized_keys_file)
 
+  connection {
+    type = "ssh"
+    user = each.value.user
+    host = each.value.ip
   }
 
+  provisioner "remote-exec" {
+    inline = [
+      templatefile("${path.module}/scripts/install-k3s-server.sh.tftpl", {
+        mode         = "agent"
+        tokens       = [random_password.k3s-server-token.result]
+        alt_names    = []
+        disable      = []
+        server_hosts = ["https://${local.support_node_ip}:6443"]
+        node_taints  = each.value.taints
+        datastores   = []
+      })
+    ]
+  }
+
+}
+