commitdev · bmonkman · Apr 14, 2020 · Apr 8, 2020 · Apr 8, 2020 · Apr 9, 2020
diff --git a/README.md b/README.md
@@ -38,7 +38,6 @@ EC2 instance sizing can be configured in [terraform/environments/staging/main.tf
 ## AWS Setting the Kubernetes context
 ```
 aws eks update-kubeconfig --name <cluster-name> --region us-west-2
-aws eks update-kubeconfig --name <cluster-name> --region us-west-2 --role-arn <role-arn>
 ```
 
 ## Workin with Kubernetes

diff --git a/kubernetes/terraform/environments/development/main.tf b/kubernetes/terraform/environments/development/main.tf
@@ -16,26 +16,8 @@ module "kubernetes" {
   region      = "<% index .Params `region` %>"
 
   # Authenticate with the EKS cluster via the cluster id
-  cluster_name = "<% .Name %>-dev-cluster"
-
-  # Assume-role policy used by monitoring fluentd daemonset
-  assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json
+  cluster_name = "<% .Name %>-dev-<% index .Params `region` %>"
 
   external_dns_zone = "<% index .Params `stagingHost` %>"
   external_dns_owner_id = "<% GenerateUUID %>" # randomly generated ID
-  external_dns_assume_roles = [ "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/k8s-staging-workers" ]
-}
-
-# Data sources for EKS IAM
-data "aws_caller_identity" "current" {}
-
-data "aws_iam_policy_document" "assumerole_root_policy" {
-  statement {
-    actions = ["sts:AssumeRole"]
-
-    principals {
-      type        = "AWS"
-      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
-    }
-  }
 }
diff --git a/kubernetes/terraform/environments/production/main.tf b/kubernetes/terraform/environments/production/main.tf
@@ -16,26 +16,8 @@ module "kubernetes" {
   region      = "<% index .Params `region` %>"
 
   # Authenticate with the EKS cluster via the cluster id
-  cluster_name = "<% .Name %>-production-cluster"
-
-  # Assume-role policy used by monitoring fluentd daemonset
-  assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json
+  cluster_name = "<% .Name %>-production-<% index .Params `region` %>"
 
   external_dns_zone = "<% index .Params `productionHost` %>"
   external_dns_owner_id = "<% GenerateUUID %>" # randomly generated ID
-  external_dns_assume_roles = [ "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/k8s-production-workers" ]
-}
-
-# Data sources for EKS IAM
-data "aws_caller_identity" "current" {}
-
-data "aws_iam_policy_document" "assumerole_root_policy" {
-  statement {
-    actions = ["sts:AssumeRole"]
-
-    principals {
-      type        = "AWS"
-      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
-    }
-  }
 }
diff --git a/kubernetes/terraform/environments/staging/main.tf b/kubernetes/terraform/environments/staging/main.tf
@@ -20,26 +20,8 @@ module "kubernetes" {
   region      = "<% index .Params `region` %>"
 
   # Authenticate with the EKS cluster via the cluster id
-  cluster_name = "<% .Name %>-staging-cluster"
-
-  # Assume-role policy used by monitoring fluentd daemonset
-  assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json
+  cluster_name = "<% .Name %>-staging-<% index .Params `region` %>"
 
   external_dns_zone = "<% index .Params `stagingHost` %>"
   external_dns_owner_id = "<% GenerateUUID %>" # randomly generated ID
-  external_dns_assume_roles = [ "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/k8s-<% .Name %>-staging-<% index .Params `region` %>-workers" ]
-}
-
-# Data sources for EKS IAM
-data "aws_caller_identity" "current" {}
-
-data "aws_iam_policy_document" "assumerole_root_policy" {
-  statement {
-    actions = ["sts:AssumeRole"]
-
-    principals {
-      type        = "AWS"
-      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
-    }
-  }
 }
diff --git a/kubernetes/terraform/modules/kubernetes/README.md b/kubernetes/terraform/modules/kubernetes/README.md
@@ -4,24 +4,30 @@
 
 This Terraform module contains configuration to provision kubernetes resources.
 
+## Core Components
+
+[Nginx Ingress Controller](https://github.com/kubernetes/ingress-nginx/)
+Use Nginx as a reverse proxy and load balancer for your cluster. This will create an AWS load balancer (ELB/ALB/NLB) and whenever an ingress is created to route traffic to your application, the controller will make sure the LB is up to date and sending traffic where it needs to go.
+
+[External DNS](https://github.com/kubernetes-sigs/external-dns)
+For any ingresses that are added to route traffic for hosts, external-dns will automatically create DNS records for those hosts and point it to the LB created by the ingress controller.
+This makes is extremely easy to bring up a new site at a specific domain or subdomain.
+
+[Cert Manager](https://github.com/jetstack/cert-manager)
+For any ingresses that specify that they need TLS, cert-manager will automatically provision a certificate using Lets Encrypt, and handle renewing it automatically on a regular basis.
+Alongside external-dns, this allows you to make sure your new domains are always secured using HTTPS.
+
+[Cloudwatch Agent/Fluentd](https://github.com/fluent/fluentd)
+A unified logging layer, Fluentd handles capturing all log output from your cluster and routing it to various sources like Cloudwatch, Elasticsearch, etc.
+
+
+...
+
 ## Organization
 
 ```
     main.tf - Configuration entrypoint.
-    ingress/ - Confioguration required to provision nginx-ingress-controller.
-        main.tf
-        provider.tf
-        variables.tf
-    monitoring/ - Configuration required to provision cluster monitoring.
-        main.tf
-        provider.tf
-        variables.tf
-        fluentd/
-            main.tf
-            files/
-                ...
-        cloudwatch/
-            main.tf
-            files/
-                ...
-```
+    external_dns.tf - Set up external-dns
+    ingress/ - Provision nginx-ingress-controller.
+    monitoring/ - Provision cluster monitoring (cloudwatch agent and fluentd).
+```
diff --git a/kubernetes/terraform/modules/kubernetes/external_dns.tf b/kubernetes/terraform/modules/kubernetes/external_dns.tf
@@ -1,28 +1,18 @@
-# Trust relationship
-data "aws_iam_policy_document" "external_dns_trust_relationship" {
-  statement {
-    actions = ["sts:AssumeRole"]
-
-    principals {
-      type        = "Service"
-      identifiers = ["ec2.amazonaws.com"]
-    }
-  }
-
-  statement {
-    actions = ["sts:AssumeRole"]
-
-    principals {
-      type        = "AWS"
-      identifiers = var.external_dns_assume_roles
-    }
-  }
+# Create a role using oidc to map service accounts
+module "iam_assumable_role" {
+  source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
+  version                       = "~> v2.6.0"
+  create_role                   = true
+  role_name                     = "<% .Name %>-k8s-${var.environment}-external-dns"
+  provider_url                  = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "")
+  role_policy_arns              = [aws_iam_policy.external_dns.arn]
+  oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:external-dns"]
 }
 
-# external-dns role
-resource "aws_iam_role" "external_dns_role" {
-  name = "k8s-external-dns-role"
-  assume_role_policy = data.aws_iam_policy_document.external_dns_trust_relationship.json
+resource "aws_iam_policy" "external_dns" {
+  name_prefix = "external-dns"
+  description = "EKS external-dns policy for cluster ${var.cluster_name}"
+  policy      = data.aws_iam_policy_document.external_dns_policy_doc.json
 }
 
 data "aws_iam_policy_document" "external_dns_policy_doc" {
@@ -48,16 +38,13 @@ data "aws_iam_policy_document" "external_dns_policy_doc" {
   }
 }
 
-resource "aws_iam_role_policy" "external_dns_policy" {
-  name = "k8s-external-dns-policy"
-  role = aws_iam_role.external_dns_role.id
-  policy = data.aws_iam_policy_document.external_dns_policy_doc.json
-}
-
 resource "kubernetes_service_account" "external_dns" {
   metadata {
-    name      = "external-dns"
-    namespace = "kube-system"
+    name        = "external-dns"
+    namespace   = "kube-system"
+    annotations = {
+      "eks.amazonaws.com/role-arn" = module.iam_assumable_role.this_iam_role_arn
+    }
   }
 }
 
@@ -116,25 +103,29 @@ resource "kubernetes_deployment" "external_dns" {
           "app"    = "external-dns",
         }
         annotations = {
-          "iam.amazonaws.com/role" = "k8s-external-dns-role",
+          "eks.amazonaws.com/role-arn" = module.iam_assumable_role.this_iam_role_arn
         }
       }
       spec {
         container {
           name  = "external-dns"
           image = "registry.opensource.zalan.do/teapot/external-dns:latest"
           args = [
-            "--source=service",
             "--source=ingress",
             "--domain-filter=${var.external_dns_zone}", # Give access only to the specified zone
             "--provider=aws",
             "--aws-zone-type=public",
             "--policy=upsert-only", # Prevent ExternalDNS from deleting any records
             "--registry=txt",
-            "--txt-owner-id=${var.external_dns_owner_id}", # ID of txt record to manage state
+            "--txt-owner-id=${var.cluster_name}", # ID of txt record to manage state
+            "--aws-batch-change-size=2", # Set the batch size to 2 so that a single record failure won't block other updates
           ]
         }
 
+        security_context {
+          fs_group =  65534
+        }
+
         service_account_name = "external-dns"
         automount_service_account_token  = true
       }

diff --git a/kubernetes/terraform/modules/kubernetes/kube2iam/README.md b/kubernetes/terraform/modules/kubernetes/kube2iam/README.md
diff --git a/kubernetes/terraform/modules/kubernetes/kube2iam/main.tf b/kubernetes/terraform/modules/kubernetes/kube2iam/main.tf
diff --git a/kubernetes/terraform/modules/kubernetes/kube2iam/variables.tf b/kubernetes/terraform/modules/kubernetes/kube2iam/variables.tf
diff --git a/kubernetes/terraform/modules/kubernetes/main.tf b/kubernetes/terraform/modules/kubernetes/main.tf
@@ -1,14 +1,8 @@
-module "kube2iam" {
-  source      = "./kube2iam"
-  environment = var.environment
-  region      = var.region
-}
 
 module "monitoring" {
   source             = "./monitoring"
   environment        = var.environment
   region             = var.region
-  assume_role_policy = var.assume_role_policy
   cluster_name       = var.cluster_name
 }