Merge branch 'share-host-key'

README.md

@@ -35,6 +35,10 @@ Any changes to the S3 bucket will be synchronised within 5 minutes
 | tags_lb | Tags to apply to the bastion load balancer | map | no | `{}` |
 | tags_asg | Tags to apply to the bastion autoscaling group | map | no | `{}` |
 | tags_sg | Tags to apply to the bastion security groups | map | no | `{}` |
+| tags_host_key | Tags to apply to the bastion host key secret and KMS key | map | no | `{}` |
+| extra_userdata | Extra commands to append to the instance user data script | string | no | |
+| log_group_name | The name of a CloudWatch log group to send logs of SSH logins and user/key changes to | string | no | |
+| s3_access_log_expiration_days | Days to keep S3 access logs, defaults to forever | number | no | |
 
 ### DNS Config
 

dns.tf

@@ -1,8 +1,8 @@
 resource "aws_route53_record" "dns_record" {
-  count = var.dns_config != null ? count(local.dns_record_types) : 0
+  count = var.dns_config != null ? length(local.dns_record_types) : 0
 
-  name    = var.dns_config.record_name
-  zone_id = var.dns_config.hosted_zone_name
+  name    = var.dns_config.domain
+  zone_id = var.dns_config.zone_id
   type    = local.dns_record_types[count.index]
 
   alias {

elb.tf

@@ -6,22 +6,22 @@ resource "aws_lb" "bastion" {
   subnets = var.public_subnet_ids
 
   load_balancer_type = "network"
-  tags               = merge({"Name" = "${var.name_prefix}lb"}, var.tags_default, var.tags_lb)
+  tags               = merge({ "Name" = "${var.name_prefix}lb" }, var.tags_default, var.tags_lb)
 }
 
 resource "aws_lb_target_group" "bastion_default" {
-  vpc_id = var.vpc_id
-
-  port        = var.external_ssh_port
-  protocol    = "TCP"
-  target_type = "instance"
+  vpc_id             = var.vpc_id
+  port               = var.external_ssh_port
+  protocol           = "TCP"
+  target_type        = "instance"
+  preserve_client_ip = true
 
   health_check {
-    port     = "traffic-port"
+    port     = 2345
     protocol = "TCP"
   }
 
-  tags = merge({"Name" = "${var.name_prefix}lb"}, var.tags_default, var.tags_lb)
+  tags = merge({ "Name" = "${var.name_prefix}lb" }, var.tags_default, var.tags_lb)
 }
 
 resource "aws_lb_listener" "bastion_ssh" {
@@ -33,4 +33,4 @@ resource "aws_lb_listener" "bastion_ssh" {
     target_group_arn = aws_lb_target_group.bastion_default.arn
     type             = "forward"
   }
-}
+}

host_key_secret.tf

@@ -0,0 +1,21 @@
+resource "aws_kms_key" "bastion_host_key_encryption_key" {
+  description         = "${var.name_prefix}bastion-host-key-kms-key"
+  enable_key_rotation = true
+  tags                = merge(var.tags_default, var.tags_host_key)
+}
+
+resource "aws_secretsmanager_secret" "bastion_host_key" {
+  name_prefix = "${var.name_prefix}bastion-ssh-host-key-"
+  description = "SSH Host key for bastion"
+  kms_key_id  = aws_kms_key.bastion_host_key_encryption_key.id
+  tags        = merge(var.tags_default, var.tags_host_key)
+}
+
+resource "aws_secretsmanager_secret_version" "bastion_host_key" {
+  secret_id     = aws_secretsmanager_secret.bastion_host_key.id
+  secret_string = tls_private_key.bastion_host_key.private_key_openssh
+}
+
+resource "tls_private_key" "bastion_host_key" {
+  algorithm = "ED25519"
+}

iam.tf

@@ -27,19 +27,41 @@ data "aws_iam_policy_document" "bastion_policy" {
     actions   = ["s3:ListBucket"]
     resources = [aws_s3_bucket.ssh_keys.arn]
   }
+
+  # Allow reading the host key secret
+  statement {
+    actions   = ["secretsmanager:GetSecretValue"]
+    resources = [aws_secretsmanager_secret.bastion_host_key.arn]
+  }
+
+  # Allow use of the KMS key used to encrypt the host key secret
+  statement {
+    actions   = ["kms:Decrypt"]
+    resources = [aws_kms_key.bastion_host_key_encryption_key.arn]
+  }
 }
 
 resource "aws_iam_policy" "bastion" {
   name_prefix = "${var.name_prefix}bastion"
   policy      = data.aws_iam_policy_document.bastion_policy.json
 }
 
-resource "aws_iam_role_policy_attachment" "lambda_given_policy" {
+resource "aws_iam_role_policy_attachment" "bastion_policy" {
   role       = aws_iam_role.bastion.name
   policy_arn = aws_iam_policy.bastion.arn
 }
 
+data "aws_iam_policy" "cloudwatch_agent" {
+  arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
+}
+
+resource "aws_iam_role_policy_attachment" "cloudwatch_agent" {
+  count      = var.log_group_name == null ? 0 : 1
+  role       = aws_iam_role.bastion.name
+  policy_arn = data.aws_iam_policy.cloudwatch_agent.arn
+}
+
 resource "aws_iam_instance_profile" "bastion_host_profile" {
   name_prefix = "${var.name_prefix}bastion-profile"
   role        = aws_iam_role.bastion.name
-}
+}

init.sh

@@ -3,11 +3,38 @@
 set -xe
 
 yum -y update --security
-yum -y install jq
+yum -y install jq nc amazon-cloudwatch-agent iptables-services
 
 mkdir /usr/bin/bastion
 mkdir /var/log/bastion
 
+systemctl enable iptables
+systemctl start iptables
+# Block non-root users from accessing the instance metadata service
+iptables -A OUTPUT -m owner ! --uid-owner root -d 169.254.169.254 -j DROP
+# Allow port 2345 for health checks
+iptables -I INPUT -p tcp -m state --state NEW -m tcp --dport 2345 -j ACCEPT
+service iptables save
+
+# Fetch the host key from AWS Secrets Manager
+aws secretsmanager get-secret-value --region ${region} --secret-id ${host_key_secret_id} --query SecretString --output text > /etc/ssh/ssh_host_ed25519_key
+ssh-keygen -y -f /etc/ssh/ssh_host_ed25519_key > /etc/ssh/ssh_host_ed25519_key.pub
+chmod 600 /etc/ssh/ssh_host_ed25519_key
+
+sed -i 's|HostKey /etc/ssh/ssh_host_ecdsa_key|#HostKey /etc/ssh/ssh_host_ecdsa_key|' /etc/ssh/sshd_config
+sed -i 's|HostKey /etc/ssh/ssh_host_rsa_key|#HostKey /etc/ssh/ssh_host_rsa_key|' /etc/ssh/sshd_config
+rm -f /etc/ssh/ssh_host_rsa_key /etc/ssh/ssh_host_rsa_key.pub
+rm -f /etc/ssh/ssh_host_ecdsa_key /etc/ssh/ssh_host_ecdsa_key.pub
+
+
+# Check the SSH config is valid, otherwise sshd will not come back up
+/usr/sbin/sshd -t
+systemctl restart sshd
+
+if [ ! -z "${cloudwatch_config_ssm_parameter}" ]; then
+  amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:${cloudwatch_config_ssm_parameter}"
+fi
+
 cat > /usr/bin/bastion/sync_users_with_s3 <<'EOF'
 #!/usr/bin/env bash
 
@@ -48,8 +75,8 @@ for row in $(cat "$S3_DATA_FILE" | jq -r '.[] | @base64'); do
   USER_NAME=$(parse_username "$KEY")
   ETAG=$(_jq '.ETag')
 
-  # Check the username starts with a letter and only contains letters, numbers and dashes afterwards
-  if [[ "$USER_NAME" =~ ^[a-z][-a-z0-9]*$ ]]; then
+  # Check the username starts with a letter and only contains letters, numbers, dashes and underscores afterwards
+  if [[ "$USER_NAME" =~ ^[a-z][-a-z0-9_]*$ ]]; then
     # Check whether the user already exists
     cut -d: -f1 /etc/passwd | grep -qx $USER_NAME  || error_code=$?
     if [ $error_code -eq 1 ]; then
@@ -101,11 +128,16 @@ fi
 EOF
 
 chmod 700 /usr/bin/bastion/sync_users_with_s3
+PATH=$PATH:/sbin /usr/bin/bastion/sync_users_with_s3
 
 # Update users every 5 minutes, check for security updates at 3AM
 cat > ~/crontab << EOF
 */5 * * * * PATH=$PATH:/sbin /usr/bin/bastion/sync_users_with_s3
 0 3 * * * yum -y update --security
+@reboot bash -c "cat /dev/null | nohup nc -kl 2345 >/dev/null 2>&1 &"
 EOF
 crontab ~/crontab
-rm ~/crontab
+rm ~/crontab
+
+# Listen on port 2345 for healthcheck pings from the load balancer
+bash -c "cat /dev/null | nohup nc -kl 2345 >/dev/null 2>&1 &"

instance.tf

@@ -18,47 +18,71 @@ data "aws_ami" "aws_linux_2" {
 
 resource "aws_security_group" "bastion" {
   description = "Enable SSH access to the bastion host from external via SSH port"
-  name        = "${var.name_prefix}main"
+  name        = "${var.name_prefix}bastion-sg"
   vpc_id      = var.vpc_id
 
-  tags = merge({ "Name" = "${var.name_prefix}main" }, var.tags_default, var.tags_sg)
+  tags = merge({ "Name" = "${var.name_prefix}bastion-sg" }, var.tags_default, var.tags_sg)
 
-  # Incoming traffic from the internet. Only allow SSH connections
-  ingress {
-    description = "Incoming SSH traffic from allowlisted CIDRs"
-    from_port   = var.external_ssh_port
-    to_port     = var.external_ssh_port
-    protocol    = "TCP"
-    cidr_blocks = concat(data.aws_subnet.subnets.*.cidr_block, var.external_allowed_cidrs)
+  lifecycle {
+    create_before_destroy = true
   }
+}
 
-  # Outgoing traffic - anything VPC only
-  egress {
-    description = "Egress - VPC only"
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    cidr_blocks = [data.aws_vpc.bastion.cidr_block]
-  }
+# Incoming traffic from the internet. Only allow SSH connections
+resource "aws_security_group_rule" "ssh_ingress" {
+  security_group_id = aws_security_group.bastion.id
+  type              = "ingress"
+  description       = "Incoming SSH traffic from allowlisted CIDRs"
+  from_port         = var.external_ssh_port
+  to_port           = var.external_ssh_port
+  protocol          = "TCP"
+  cidr_blocks       = concat(data.aws_subnet.public_subnets.*.cidr_block, var.external_allowed_cidrs)
+}
 
-  # Plus allow HTTP(S) internet egress for yum updates
-  # tfsec:ignore:aws-vpc-no-public-egress-sgr
-  egress {
-    description = "Outbound TLS"
-    from_port   = 443
-    to_port     = 443
-    protocol    = "tcp"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
+# Health checks
+resource "aws_security_group_rule" "health_check" {
+  security_group_id = aws_security_group.bastion.id
+  type              = "ingress"
+  description       = "Health check"
+  from_port         = 2345
+  to_port           = 2345
+  protocol          = "TCP"
+  cidr_blocks       = data.aws_subnet.public_subnets.*.cidr_block
+}
 
-  # tfsec:ignore:aws-vpc-no-public-egress-sgr
-  egress {
-    description = "Outbound HTTP"
-    from_port   = 80
-    to_port     = 80
-    protocol    = "tcp"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
+# Outgoing traffic - anything VPC only
+resource "aws_security_group_rule" "vpc_egress" {
+  security_group_id = aws_security_group.bastion.id
+  type              = "egress"
+  description       = "Egress - VPC only"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "-1"
+  cidr_blocks       = [data.aws_vpc.bastion.cidr_block]
+}
+
+
+# Plus allow HTTP(S) internet egress for yum updates
+# tfsec:ignore:aws-vpc-no-public-egress-sgr
+resource "aws_security_group_rule" "https_egress" {
+  security_group_id = aws_security_group.bastion.id
+  type              = "egress"
+  description       = "Outbound TLS"
+  from_port         = 443
+  to_port           = 443
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+# tfsec:ignore:aws-vpc-no-public-egress-sgr
+resource "aws_security_group_rule" "http_egress" {
+  security_group_id = aws_security_group.bastion.id
+  type              = "egress"
+  description       = "Outbound HTTP"
+  from_port         = 80
+  to_port           = 80
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
 }
 
 resource "aws_launch_configuration" "bastion" {
@@ -73,10 +97,15 @@ resource "aws_launch_configuration" "bastion" {
 
   security_groups = [aws_security_group.bastion.id]
 
-  user_data = templatefile("${path.module}/init.sh", {
-    region      = var.region
-    bucket_name = aws_s3_bucket.ssh_keys.bucket
-  })
+  user_data = join("\n", [
+    templatefile("${path.module}/init.sh", {
+      region                          = var.region
+      bucket_name                     = aws_s3_bucket.ssh_keys.bucket
+      host_key_secret_id              = aws_secretsmanager_secret_version.bastion_host_key.secret_id
+      cloudwatch_config_ssm_parameter = var.log_group_name == null ? "" : aws_ssm_parameter.cloudwatch_agent_config[0].name
+    }),
+    var.extra_userdata
+  ])
 
   root_block_device {
     encrypted = true
@@ -95,13 +124,13 @@ resource "aws_launch_configuration" "bastion" {
 resource "aws_autoscaling_group" "bastion" {
   name_prefix          = "${var.name_prefix}asg-"
   launch_configuration = aws_launch_configuration.bastion.name
-  max_size             = local.instance_count
+  max_size             = local.instance_count + 1
   min_size             = local.instance_count
   desired_capacity     = local.instance_count
 
   vpc_zone_identifier = var.instance_subnet_ids
 
-  default_cooldown          = 180
+  default_cooldown          = 30
   health_check_grace_period = 180
   health_check_type         = "EC2"
 
@@ -114,14 +143,18 @@ resource "aws_autoscaling_group" "bastion" {
   ]
 
   dynamic "tag" {
-    for_each = merge({ "Name" = "${var.name_prefix}asg" }, var.tags_default, var.tags_asg)
+    for_each = merge({ "Name" = "${var.name_prefix}bastion-instances-asg" }, var.tags_default, var.tags_asg)
     content {
       key                 = tag.key
       value               = tag.value
       propagate_at_launch = true
     }
   }
 
+  instance_refresh {
+    strategy = "Rolling"
+  }
+
   lifecycle {
     create_before_destroy = true
   }

logging.tf

@@ -0,0 +1,30 @@
+resource "aws_ssm_parameter" "cloudwatch_agent_config" {
+  count = var.log_group_name == null ? 0 : 1
+
+  # CloudWatchAgentServerPolicy grants permission to read parameters with the prefix "AmazonCloudWatch-"
+  name = "AmazonCloudWatch-${var.name_prefix}-bastion"
+  type = "String"
+  value = jsonencode({
+    agent = {
+      metrics_collection_interval = 5
+    },
+    logs = {
+      logs_collected = {
+        files = {
+          collect_list = [
+            {
+              log_group_name  = var.log_group_name
+              log_stream_name = "{instance_id}-ssh"
+              file_path       = "/var/log/secure"
+            },
+            {
+              log_group_name  = var.log_group_name
+              log_stream_name = "{instance_id}-changelog"
+              file_path       = "/var/log/bastion/changelog.log"
+            }
+          ]
+        }
+      }
+    }
+  })
+}

main.tf

@@ -9,7 +9,7 @@ data "aws_vpc" "bastion" {
   id = var.vpc_id
 }
 
-data "aws_subnet" "subnets" {
+data "aws_subnet" "public_subnets" {
   count = length(var.public_subnet_ids)
   id    = var.public_subnet_ids[count.index]
 }