Ship SSH and key change logs to CloudWatch

* Optional new variable for log group name
* Install and configure the CloudWatch agent
* Move the healthchecks to a different port so the NLB doesn't fill up
  the SSH logs

README.md

@@ -36,6 +36,7 @@ Any changes to the S3 bucket will be synchronised within 5 minutes
 | tags_asg | Tags to apply to the bastion autoscaling group | map | no | `{}` |
 | tags_sg | Tags to apply to the bastion security groups | map | no | `{}` |
 | extra_userdata | Extra commands to append to the instance user data script | string | no | |
+| log_group_name | The name of a CloudWatch log group to send logs of SSH logins and user/key changes to | string | no | |
 
 ### DNS Config
 

elb.tf

@@ -10,14 +10,14 @@ resource "aws_lb" "bastion" {
 }
 
 resource "aws_lb_target_group" "bastion_default" {
-  vpc_id = var.vpc_id
-
-  port        = var.external_ssh_port
-  protocol    = "TCP"
-  target_type = "instance"
+  vpc_id             = var.vpc_id
+  port               = var.external_ssh_port
+  protocol           = "TCP"
+  target_type        = "instance"
+  preserve_client_ip = true
 
   health_check {
-    port     = "traffic-port"
+    port     = 2345
     protocol = "TCP"
   }
 
@@ -33,4 +33,4 @@ resource "aws_lb_listener" "bastion_ssh" {
     target_group_arn = aws_lb_target_group.bastion_default.arn
     type             = "forward"
   }
-}
+}

iam.tf

@@ -51,6 +51,16 @@ resource "aws_iam_role_policy_attachment" "bastion_policy" {
   policy_arn = aws_iam_policy.bastion.arn
 }
 
+data "aws_iam_policy" "cloudwatch_agent" {
+  arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
+}
+
+resource "aws_iam_role_policy_attachment" "cloudwatch_agent" {
+  count      = var.log_group_name == null ? 0 : 1
+  role       = aws_iam_role.bastion.name
+  policy_arn = data.aws_iam_policy.cloudwatch_agent.arn
+}
+
 resource "aws_iam_instance_profile" "bastion_host_profile" {
   name_prefix = "${var.name_prefix}bastion-profile"
   role        = aws_iam_role.bastion.name

init.sh

@@ -3,13 +3,18 @@
 set -xe
 
 yum -y update --security
-yum -y install jq
+yum -y install jq nc amazon-cloudwatch-agent iptables-services
 
 mkdir /usr/bin/bastion
 mkdir /var/log/bastion
 
+systemctl enable iptables
+systemctl start iptables
 # Block non-root users from accessing the instance metadata service
 iptables -A OUTPUT -m owner ! --uid-owner root -d 169.254.169.254 -j DROP
+# Allow port 2345 for health checks
+iptables -I INPUT -p tcp -m state --state NEW -m tcp --dport 2345 -j ACCEPT
+service iptables save
 
 # Fetch the host key from AWS Secrets Manager
 aws secretsmanager get-secret-value --region ${region} --secret-id ${host_key_secret_id} --query SecretString --output text > /etc/ssh/ssh_host_ed25519_key
@@ -26,6 +31,10 @@ rm -f /etc/ssh/ssh_host_ecdsa_key /etc/ssh/ssh_host_ecdsa_key.pub
 /usr/sbin/sshd -t
 systemctl restart sshd
 
+if [ ! -z "${cloudwatch_config_ssm_parameter}" ]; then
+  amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c "ssm:${cloudwatch_config_ssm_parameter}"
+fi
+
 cat > /usr/bin/bastion/sync_users_with_s3 <<'EOF'
 #!/usr/bin/env bash
 
@@ -119,11 +128,16 @@ fi
 EOF
 
 chmod 700 /usr/bin/bastion/sync_users_with_s3
+PATH=$PATH:/sbin /usr/bin/bastion/sync_users_with_s3
 
 # Update users every 5 minutes, check for security updates at 3AM
 cat > ~/crontab << EOF
 */5 * * * * PATH=$PATH:/sbin /usr/bin/bastion/sync_users_with_s3
 0 3 * * * yum -y update --security
+@reboot bash -c "cat /dev/null | nohup nc -kl 2345 >/dev/null 2>&1 &"
 EOF
 crontab ~/crontab
 rm ~/crontab
+
+# Listen on port 2345 for healthcheck pings from the load balancer
+bash -c "cat /dev/null | nohup nc -kl 2345 >/dev/null 2>&1 &"

instance.tf

@@ -36,7 +36,18 @@ resource "aws_security_group_rule" "ssh_ingress" {
   from_port         = var.external_ssh_port
   to_port           = var.external_ssh_port
   protocol          = "TCP"
-  cidr_blocks       = concat(data.aws_subnet.subnets.*.cidr_block, var.external_allowed_cidrs)
+  cidr_blocks       = concat(data.aws_subnet.public_subnets.*.cidr_block, var.external_allowed_cidrs)
+}
+
+# Health checks
+resource "aws_security_group_rule" "health_check" {
+  security_group_id = aws_security_group.bastion.id
+  type              = "ingress"
+  description       = "Health check"
+  from_port         = 2345
+  to_port           = 2345
+  protocol          = "TCP"
+  cidr_blocks       = data.aws_subnet.public_subnets.*.cidr_block
 }
 
 # Outgoing traffic - anything VPC only
@@ -88,9 +99,10 @@ resource "aws_launch_configuration" "bastion" {
 
   user_data = join("\n", [
     templatefile("${path.module}/init.sh", {
-      region             = var.region
-      bucket_name        = aws_s3_bucket.ssh_keys.bucket,
-      host_key_secret_id = aws_secretsmanager_secret_version.bastion_host_key.secret_id,
+      region                          = var.region
+      bucket_name                     = aws_s3_bucket.ssh_keys.bucket
+      host_key_secret_id              = aws_secretsmanager_secret_version.bastion_host_key.secret_id
+      cloudwatch_config_ssm_parameter = var.log_group_name == null ? "" : aws_ssm_parameter.cloudwatch_agent_config[0].name
     }),
     var.extra_userdata
   ])
@@ -112,7 +124,7 @@ resource "aws_launch_configuration" "bastion" {
 resource "aws_autoscaling_group" "bastion" {
   name_prefix          = "${var.name_prefix}asg-"
   launch_configuration = aws_launch_configuration.bastion.name
-  max_size             = local.instance_count
+  max_size             = local.instance_count + 1
   min_size             = local.instance_count
   desired_capacity     = local.instance_count
 

logging.tf

@@ -0,0 +1,30 @@
+resource "aws_ssm_parameter" "cloudwatch_agent_config" {
+  count = var.log_group_name == null ? 0 : 1
+
+  # CloudWatchAgentServerPolicy grants permission to read parameters with the prefix "AmazonCloudWatch-"
+  name = "AmazonCloudWatch-${var.name_prefix}-bastion"
+  type = "String"
+  value = jsonencode({
+    agent = {
+      metrics_collection_interval = 5
+    },
+    logs = {
+      logs_collected = {
+        files = {
+          collect_list = [
+            {
+              log_group_name  = var.log_group_name
+              log_stream_name = "{instance_id}-ssh"
+              file_path       = "/var/log/secure"
+            },
+            {
+              log_group_name  = var.log_group_name
+              log_stream_name = "{instance_id}-changelog"
+              file_path       = "/var/log/bastion/changelog.log"
+            }
+          ]
+        }
+      }
+    }
+  })
+}

main.tf

@@ -9,7 +9,7 @@ data "aws_vpc" "bastion" {
   id = var.vpc_id
 }
 
-data "aws_subnet" "subnets" {
+data "aws_subnet" "public_subnets" {
   count = length(var.public_subnet_ids)
   id    = var.public_subnet_ids[count.index]
 }

variables.tf

@@ -92,3 +92,10 @@ variable "extra_userdata" {
   default     = ""
   description = "Extra commands to append to the instance user data script"
 }
+
+
+variable "log_group_name" {
+  type        = string
+  default     = null
+  description = "Optional log group to send SSH logs to"
+}