257 add vpn support (#85)

sshi100 · bmonkman · web-flow · commit 5a5b78700aa7 · 2020-09-11T14:13:42.000-07:00
* initial submit

* set initial sample client public keys

* add random_seed

* fix folder

* enhancement with review

* run terraform fmt against related directories

* add hostname vpn.* pointing to service NLB

* formalized configuration

* added app client configuration retrieval script and initial README

* added local script to help user adding and configuration generation

* added local script to help user adding and configuration generation

* use template for PROJECT

* attached k8s current context with config file name

* revision on config file name

* A few tweaks to documentation

Co-authored-by: Bill Monkman &lt;bmonkman@gmail.com&gt;
diff --git a/templates/Makefile b/templates/Makefile
@@ -1,7 +1,7 @@
 ENVIRONMENT ?= stage
 PROJECT = <% .Name %>
 
-apply: apply-remote-state apply-secrets apply-env apply-k8s-utils post-apply-setup
+apply: apply-remote-state apply-secrets apply-env update-k8s-conf pre-k8s apply-k8s-utils post-apply-setup
 
 apply-remote-state:
 	aws s3 ls $(PROJECT)-$(ENVIRONMENT)-terraform-state || (\
@@ -22,7 +22,13 @@ apply-env:
 	terraform init && \
 	terraform apply $(AUTO_APPROVE)
 
-apply-k8s-utils: update-k8s-conf
+pre-k8s:
+	@echo "Creating VPN private key..."
+	WGKEY=$(shell kubectl run -i --tty zero-k8s-utilities --image=commitdev/zero-k8s-utilities:0.0.3 --restart=Never -- wg genkey) && kubectl delete pod/zero-k8s-utilities && \
+	aws secretsmanager create-secret --name $(PROJECT)-$(ENVIRONMENT)-vpn-wg-privatekey-<% index .Params `randomSeed` %> --description "Auto-generated Wireguard VPN private key" --secret-string $$WGKEY
+	@echo "Done VPN private key creation"
+
+apply-k8s-utils:
 	cd kubernetes/terraform/environments/$(ENVIRONMENT) && \
 	terraform init && \
 	terraform apply $(AUTO_APPROVE)
diff --git a/templates/kubernetes/terraform/environments/dev/application_iam_policy.tf b/templates/kubernetes/terraform/environments/dev/application_iam_policy.tf
@@ -10,4 +10,4 @@ data "aws_iam_policy_document" "resource_access_backendservice" {
     resources = ["arn:aws:ec2:::dev-*"]
   }
   # can be more statements here
-}
+}
diff --git a/templates/kubernetes/terraform/environments/dev/main.tf b/templates/kubernetes/terraform/environments/dev/main.tf
@@ -16,11 +16,12 @@ module "kubernetes" {
 
   environment = "dev"
   region      = "<% index .Params `region` %>"
+  random_seed = "<% index .Params `randomSeed` %>"
 
   # Authenticate with the EKS cluster via the cluster id
   cluster_name = "<% .Name %>-dev-<% index .Params `region` %>"
 
-  external_dns_zone = "<% index .Params `stagingHostRoot` %>"
+  external_dns_zone     = "<% index .Params `stagingHostRoot` %>"
   external_dns_owner_id = "<% GenerateUUID %>" # randomly generated ID
 
   # Registration email for LetsEncrypt
@@ -38,4 +39,11 @@ module "kubernetes" {
     }
     # could be more policies defined here (if have)
   ]
+
+  # Wireguard configuration
+  vpn_server_address = "10.10.254.0/24"
+  vpn_client_publickeys = [
+    ["Max C", "10.10.254.201/32", "/B3Q/Hlf+ILInjpehTLk9DZGgybdGdbm0SsG87OnWV0="],
+    ["Carter L", "10.10.254.202/32", "h2jMuaXNIlx7Z0a3owWFjPsAA8B+ZpQH3FbZK393+08="],
+  ]
 }
diff --git a/templates/kubernetes/terraform/environments/prod/main.tf b/templates/kubernetes/terraform/environments/prod/main.tf
@@ -9,7 +9,7 @@ terraform {
 }
 
 provider "aws" {
-  region  = "<% index .Params `region` %>"
+  region = "<% index .Params `region` %>"
 }
 
 # Provision kubernetes resources required to run services/applications
@@ -24,7 +24,7 @@ module "kubernetes" {
   # Authenticate with the EKS cluster via the cluster id
   cluster_name = "<% .Name %>-prod-<% index .Params `region` %>"
 
-  external_dns_zone = "<% index .Params `productionHostRoot` %>"
+  external_dns_zone     = "<% index .Params `productionHostRoot` %>"
   external_dns_owner_id = "<% GenerateUUID %>" # randomly generated ID
 
   # Registration email for LetsEncrypt
@@ -42,4 +42,11 @@ module "kubernetes" {
     }
     # could be more policies defined here (if have)
   ]
+
+  # Wireguard configuration
+  vpn_server_address = "10.10.99.0/24"
+  vpn_client_publickeys = [
+    ["Max C", "10.10.99.201/32", "/B3Q/Hlf+ILInjpehTLk9DZGgybdGdbm0SsG87OnWV0="],
+    ["Carter L", "10.10.99.202/32", "h2jMuaXNIlx7Z0a3owWFjPsAA8B+ZpQH3FbZK393+08="],
+  ]
 }
diff --git a/templates/kubernetes/terraform/environments/stage/application_iam_policy.tf b/templates/kubernetes/terraform/environments/stage/application_iam_policy.tf
@@ -10,4 +10,4 @@ data "aws_iam_policy_document" "resource_access_backendservice" {
     resources = ["arn:aws:ec2:::stage-*"]
   }
   # can be more statements here
-}
+}
diff --git a/templates/kubernetes/terraform/environments/stage/main.tf b/templates/kubernetes/terraform/environments/stage/main.tf
@@ -9,7 +9,7 @@ terraform {
 }
 
 provider "aws" {
-  region  = "<% index .Params `region` %>"
+  region = "<% index .Params `region` %>"
 }
 
 # Provision kubernetes resources required to run services/applications
@@ -20,11 +20,12 @@ module "kubernetes" {
 
   environment = "stage"
   region      = "<% index .Params `region` %>"
+  random_seed = "<% index .Params `randomSeed` %>"
 
   # Authenticate with the EKS cluster via the cluster id
   cluster_name = "<% .Name %>-stage-<% index .Params `region` %>"
 
-  external_dns_zone = "<% index .Params `stagingHostRoot` %>"
+  external_dns_zone     = "<% index .Params `stagingHostRoot` %>"
   external_dns_owner_id = "<% GenerateUUID %>" # randomly generated ID
 
   # Registration email for LetsEncrypt
@@ -42,4 +43,11 @@ module "kubernetes" {
     }
     # could be more policies defined here (if have)
   ]
+
+  # Wireguard configuration
+  vpn_server_address = "10.10.199.0/24"
+  vpn_client_publickeys = [
+    ["Max C", "10.10.199.201/32", "/B3Q/Hlf+ILInjpehTLk9DZGgybdGdbm0SsG87OnWV0="],
+    ["Carter L", "10.10.199.202/32", "h2jMuaXNIlx7Z0a3owWFjPsAA8B+ZpQH3FbZK393+08="],
+  ]
 }
diff --git a/templates/kubernetes/terraform/modules/kubernetes/README.md b/templates/kubernetes/terraform/modules/kubernetes/README.md
@@ -52,6 +52,29 @@ The `irsa` module makes it easy to grant a pod to have a specific level of acces
 ```
 
 
+## WireGuard VPN support
+WireGuard&reg; is an extremely simple yet fast and modern VPN that utilizes state-of-the-art cryptography. This allows users to access internal resources securely.
+
+A WireGuard pod will be started inside the cluster and users can be added to it by appending lines to `kubernetes/terraform/environments/<env>/main.tf`:
+```
+  vpn_client_publickeys = [
+    # name, IP, public key
+    ["Your Name", "10.10.199.203/32", "yz6gNspLJE/HtftBwcj5x0yK2XG6+/SHIaZ****vFRc="],
+  ]
+```
+
+A new user can add themselves to the VPN server easily. Any user with access to the kubernetes cluster should be able to run the script `scripts/add-vpn-user.sh`
+This will ask for their name, and automatically generate a line like the one above, which they can then add to the terraform and apply themselves, or give the line to an administrator and ask them to apply it.
+The environment they are added to will be decided by the current `kubectl` context. You can see your current context with `kubectl config current-context`.
+A user will need to repeat this for each environment they need access to (for example, staging and production.)
+
+*Note that this will try to detect the next available IP address for the user but you should still take care to ensure there are no duplicate IPs in the list.*
+
+It will also generate a WireGuard client config file on their local machine which will be properly populated with all the values to allow them to connect to the server.
+
+The WireGuard client can be downloaded at [https://www.wireguard.com/install/](https://www.wireguard.com/install/)
+
+Once connected to the VPN, the user should have direct access to anything running inside the AWS VPC.
 
 ## Organization
 
diff --git a/templates/kubernetes/terraform/modules/kubernetes/cert_manager.tf b/templates/kubernetes/terraform/modules/kubernetes/cert_manager.tf
@@ -70,12 +70,12 @@ resource "helm_release" "cert_manager" {
   set {
     type  = "string"
     name  = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
-    value =  module.iam_assumable_role_cert_manager.this_iam_role_arn
+    value = module.iam_assumable_role_cert_manager.this_iam_role_arn
   }
   set {
     type  = "string"
     name  = "podAnnotations.eks\\.amazonaws\\.com/role-arn"
-    value =  module.iam_assumable_role_cert_manager.this_iam_role_arn
+    value = module.iam_assumable_role_cert_manager.this_iam_role_arn
   }
   set {
     name  = "securityContext.fsGroup"
diff --git a/templates/kubernetes/terraform/modules/kubernetes/cluster_autoscaler.tf b/templates/kubernetes/terraform/modules/kubernetes/cluster_autoscaler.tf
@@ -20,7 +20,7 @@ resource "helm_release" "cluster_autoscaler" {
   set {
     type  = "string"
     name  = "rbac.serviceAccountAnnotations.eks\\.amazonaws\\.com/role-arn"
-    value =  module.iam_assumable_role_cluster_autoscaler.this_iam_role_arn
+    value = module.iam_assumable_role_cluster_autoscaler.this_iam_role_arn
   }
   set {
     name  = "awsRegion"
diff --git a/templates/kubernetes/terraform/modules/kubernetes/database_service.tf b/templates/kubernetes/terraform/modules/kubernetes/database_service.tf
@@ -8,10 +8,10 @@ resource "kubernetes_service" "app_db" {
   ## https://github.com/commitdev/zero-deployable-backend/blob/b2cee21982b1e6a0ac9996e2a1bf214e5bf10ab5/db-ops/create-db-user.sh#L6
   metadata {
     namespace = kubernetes_namespace.app_namespace.metadata[0].name
-    name = "database"
+    name      = "database"
   }
   spec {
-    type = "ExternalName"
+    type          = "ExternalName"
     external_name = data.aws_db_instance.database.address
   }
 }
diff --git a/templates/kubernetes/terraform/modules/kubernetes/external_dns.tf b/templates/kubernetes/terraform/modules/kubernetes/external_dns.tf
@@ -40,8 +40,8 @@ data "aws_iam_policy_document" "external_dns_policy_doc" {
 
 resource "kubernetes_service_account" "external_dns" {
   metadata {
-    name        = "external-dns"
-    namespace   = "kube-system"
+    name      = "external-dns"
+    namespace = "kube-system"
     annotations = {
       "eks.amazonaws.com/role-arn" = module.iam_assumable_role_external_dns.this_iam_role_arn
     }
@@ -62,11 +62,16 @@ resource "kubernetes_cluster_role" "external_dns" {
     api_groups = ["extensions"]
     resources  = ["ingresses"]
   }
-rule {
+  rule {
     verbs      = ["list"]
     api_groups = [""]
     resources  = ["nodes"]
   }
+  rule {
+    verbs      = ["get", "list", "watch"]
+    api_groups = [""]
+    resources  = ["endpoints"]
+  }
 }
 
 resource "kubernetes_cluster_role_binding" "external_dns" {
@@ -94,13 +99,13 @@ resource "kubernetes_deployment" "external_dns" {
     replicas = 1
     selector {
       match_labels = {
-        "app"    = "external-dns",
+        "app" = "external-dns",
       }
     }
     template {
       metadata {
         labels = {
-          "app"    = "external-dns",
+          "app" = "external-dns",
         }
       }
       spec {
@@ -109,22 +114,23 @@ resource "kubernetes_deployment" "external_dns" {
           image = "registry.opensource.zalan.do/teapot/external-dns:latest"
           args = [
             "--source=ingress",
+            "--source=service",
             "--domain-filter=${var.external_dns_zone}", # Give access only to the specified zone
             "--provider=aws",
             "--aws-zone-type=public",
             "--policy=upsert-only", # Prevent ExternalDNS from deleting any records
             "--registry=txt",
             "--txt-owner-id=${var.cluster_name}", # ID of txt record to manage state
-            "--aws-batch-change-size=2", # Set the batch size to 2 so that a single record failure won't block other updates
+            "--aws-batch-change-size=2",          # Set the batch size to 2 so that a single record failure won't block other updates
           ]
         }
 
         security_context {
-          fs_group =  65534
+          fs_group = 65534
         }
 
-        service_account_name = "external-dns"
-        automount_service_account_token  = true
+        service_account_name            = "external-dns"
+        automount_service_account_token = true
       }
     }
   }
diff --git a/templates/kubernetes/terraform/modules/kubernetes/files/wireguard-peer-csv.tpl b/templates/kubernetes/terraform/modules/kubernetes/files/wireguard-peer-csv.tpl
@@ -0,0 +1 @@
+${tpl_client_name}|${tpl_client_ip}|${tpl_client_pub_key}
diff --git a/templates/kubernetes/terraform/modules/kubernetes/files/wireguard-peer.tpl b/templates/kubernetes/terraform/modules/kubernetes/files/wireguard-peer.tpl
@@ -0,0 +1,5 @@
+[Peer]
+#for client: ${tpl_client_name}
+PublicKey = ${tpl_client_pub_key}
+AllowedIPs = ${tpl_client_ip}
+PersistentKeepalive = 25
diff --git a/templates/kubernetes/terraform/modules/kubernetes/files/wireguard-wg0-conf.tpl b/templates/kubernetes/terraform/modules/kubernetes/files/wireguard-wg0-conf.tpl
@@ -0,0 +1,12 @@
+[Interface]
+Address = ${tpl_server_address}
+ListenPort = 51820
+PostUp = wg set wg0 private-key /etc/wireguard/privatekey
+PostUp   = iptables -A FORWARD -s ${tpl_server_address} -d ${tpl_destination_subnets} -j ACCEPT
+PostUp   = iptables -A FORWARD -s ${tpl_server_address} -j DROP
+PostUp   = iptables -t nat -A POSTROUTING -s ${tpl_server_address} -o eth0 -j MASQUERADE
+PostDown = iptables -D FORWARD -s ${tpl_server_address} -d ${tpl_destination_subnets} -j ACCEPT
+PostDown = iptables -D FORWARD -s ${tpl_server_address} -j DROP
+PostDown = iptables -t nat -D POSTROUTING -s ${tpl_server_address} -o eth0 -j MASQUERADE
+
+${tpl_client_peers}
diff --git a/templates/kubernetes/terraform/modules/kubernetes/irsa.tf b/templates/kubernetes/terraform/modules/kubernetes/irsa.tf
@@ -1,14 +1,14 @@
 # IRSA support: allow backend service to have a specific policy via service-account and role
 
 # application_policy_list is passed from main.tf as below:
-  # application_policy_list = [
-  #   {
-  #     service_account = "backend-service"
-  #     namespace       = "my-app"
-  #     policy          = data.aws_iam_policy_document.resource_access_app1
-  #   }
-  #   # could be more here
-  # ]
+# application_policy_list = [
+#   {
+#     service_account = "backend-service"
+#     namespace       = "my-app"
+#     policy          = data.aws_iam_policy_document.resource_access_app1
+#   }
+#   # could be more here
+# ]
 
 # Create a role using oidc to map service accounts
 module "iam_assumable_role_irsa" {
@@ -32,10 +32,10 @@ resource "aws_iam_policy" "irsa" {
 
 # Create kubernetes service account
 resource "kubernetes_service_account" "irsa" {
-  count         = length(var.application_policy_list)
+  count = length(var.application_policy_list)
   metadata {
-    name        = var.application_policy_list[count.index].service_account
-    namespace   = var.application_policy_list[count.index].namespace
+    name      = var.application_policy_list[count.index].service_account
+    namespace = var.application_policy_list[count.index].namespace
     annotations = {
       "eks.amazonaws.com/role-arn" = module.iam_assumable_role_irsa[count.index].this_iam_role_arn
     }
diff --git a/templates/kubernetes/terraform/modules/kubernetes/main.tf b/templates/kubernetes/terraform/modules/kubernetes/main.tf
@@ -1,10 +1,10 @@
 
 module "logging_cloudwatch" {
-  count              = var.logging_type == "cloudwatch" ? 1 : 0
-  source             = "./logging/cloudwatch"
-  environment        = var.environment
-  region             = var.region
-  cluster_name       = var.cluster_name
+  count        = var.logging_type == "cloudwatch" ? 1 : 0
+  source       = "./logging/cloudwatch"
+  environment  = var.environment
+  region       = var.region
+  cluster_name = var.cluster_name
 }
 
 module "logging_kibana" {
diff --git a/templates/kubernetes/terraform/modules/kubernetes/provider.tf b/templates/kubernetes/terraform/modules/kubernetes/provider.tf
@@ -14,14 +14,14 @@ provider "kubernetes" {
   exec {
     api_version = "client.authentication.k8s.io/v1alpha1"
     command     = "aws"
-    args        = [
+    args = [
       "eks",
       "get-token",
       "--region",
       var.region,
       "--cluster-name",
       var.cluster_name,
       "--role",
-      "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.project}-kubernetes-admin-${var.environment}"]
+    "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.project}-kubernetes-admin-${var.environment}"]
   }
 }
diff --git a/templates/kubernetes/terraform/modules/kubernetes/variables.tf b/templates/kubernetes/terraform/modules/kubernetes/variables.tf
@@ -10,6 +10,10 @@ variable "environment" {
   description = "Environment"
 }
 
+variable "random_seed" {
+  description = "A randomly generated string to prevent collisions of resource names - should be unique within an AWS account"
+}
+
 variable "cluster_name" {
   description = "Kubernetes cluster name"
 }
@@ -38,7 +42,7 @@ variable "logging_type" {
   default     = "cloudwatch"
 
   validation {
-    condition     = (
+    condition = (
       var.logging_type == "cloudwatch" || var.logging_type == "kibana"
     )
     error_message = "Invalid value. Valid values are cloudwatch or kibana."
@@ -50,3 +54,13 @@ variable "application_policy_list" {
   type        = list
   default     = []
 }
+
+variable "vpn_server_address" {
+  description = "VPN server address"
+  type        = string
+}
+
+variable "vpn_client_publickeys" {
+  type        = list(tuple([string, string, string]))
+  description = "VPN List of client name, IP and public key"
+}
diff --git a/templates/kubernetes/terraform/modules/kubernetes/vpn.tf b/templates/kubernetes/terraform/modules/kubernetes/vpn.tf
diff --git a/templates/scripts/add-vpn-user.sh b/templates/scripts/add-vpn-user.sh

Original file line number	Diff line number	Diff line change
`@@ -10,4 +10,4 @@ data "aws_iam_policy_document" "resource_access_backendservice" {`
`10`	`10`	`resources = ["arn:aws:ec2:::dev-*"]`
`11`	`11`	`}`
`12`	`12`	`# can be more statements here`
`13`		`-}`
	`13`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@ terraform {`
`9`	`9`	`}`
`10`	`10`
`11`	`11`	`provider "aws" {`
`12`		- region = "<% index .Params `region` %>"
	`12`	+ region = "<% index .Params `region` %>"
`13`	`13`	`}`
`14`	`14`
`15`	`15`	`# Provision kubernetes resources required to run services/applications`
`@@ -24,7 +24,7 @@ module "kubernetes" {`
`24`	`24`	`# Authenticate with the EKS cluster via the cluster id`
`25`	`25`	cluster_name = "<% .Name %>-prod-<% index .Params `region` %>"
`26`	`26`
`27`		- external_dns_zone = "<% index .Params `productionHostRoot` %>"
	`27`	+ external_dns_zone = "<% index .Params `productionHostRoot` %>"
`28`	`28`	`external_dns_owner_id = "<% GenerateUUID %>" # randomly generated ID`
`29`	`29`
`30`	`30`	`# Registration email for LetsEncrypt`
`@@ -42,4 +42,11 @@ module "kubernetes" {`
`42`	`42`	`}`
`43`	`43`	`# could be more policies defined here (if have)`
`44`	`44`	`]`
	`45`	`+`
	`46`	`+ # Wireguard configuration`
	`47`	`+ vpn_server_address = "10.10.99.0/24"`
	`48`	`+ vpn_client_publickeys = [`
	`49`	`+ ["Max C", "10.10.99.201/32", "/B3Q/Hlf+ILInjpehTLk9DZGgybdGdbm0SsG87OnWV0="],`
	`50`	`+ ["Carter L", "10.10.99.202/32", "h2jMuaXNIlx7Z0a3owWFjPsAA8B+ZpQH3FbZK393+08="],`
	`51`	`+ ]`
`45`	`52`	`}`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@ resource "helm_release" "cluster_autoscaler" {`
`20`	`20`	`set {`
`21`	`21`	`type = "string"`
`22`	`22`	`name = "rbac.serviceAccountAnnotations.eks\\.amazonaws\\.com/role-arn"`
`23`		`- value = module.iam_assumable_role_cluster_autoscaler.this_iam_role_arn`
	`23`	`+ value = module.iam_assumable_role_cluster_autoscaler.this_iam_role_arn`
`24`	`24`	`}`
`25`	`25`	`set {`
`26`	`26`	`name = "awsRegion"`
Original file line number	Diff line number	Diff line change
`@@ -8,10 +8,10 @@ resource "kubernetes_service" "app_db" {`
`8`	`8`	`## https://github.com/commitdev/zero-deployable-backend/blob/b2cee21982b1e6a0ac9996e2a1bf214e5bf10ab5/db-ops/create-db-user.sh#L6`
`9`	`9`	`metadata {`
`10`	`10`	`namespace = kubernetes_namespace.app_namespace.metadata[0].name`
`11`		`- name = "database"`
	`11`	`+ name = "database"`
`12`	`12`	`}`
`13`	`13`	`spec {`
`14`		`- type = "ExternalName"`
	`14`	`+ type = "ExternalName"`
`15`	`15`	`external_name = data.aws_db_instance.database.address`
`16`	`16`	`}`
`17`	`17`	`}`