commitdev · bmonkman · Apr 15, 2020 · Apr 14, 2020 · Apr 14, 2020 · Apr 14, 2020
diff --git a/kubernetes/terraform/modules/kubernetes/README.md b/kubernetes/terraform/modules/kubernetes/README.md
@@ -21,7 +21,45 @@ Alongside external-dns, this allows you to make sure your new domains are always
 A unified logging layer, Fluentd handles capturing all log output from your cluster and routing it to various sources like Cloudwatch, Elasticsearch, etc.
 
 
-...
+## AWS IAM / Kubernetes RBAC integration
+
+Sometimes you may have an application running in your cluster that needs to access the AWS API (S3 is a common example.) In this case you want to be able to have fine-grained control over this, to allow an application only the very specific access it needs.
+
+Previously there were tools like `kube2iam` or `kiam` that would enable this functionality, but now there is a new official method that AWS introduced that they call [IRSA (IAM Roles for Service Accounts)](https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/)
+
+This uses their OIDC IAM support to be able to mount tokens into pods automatically that can then be used to auth with the AWS API using a specific role.
+
+The `cert_manager.tf` config has a good example of using this in practice. To allow a pod to have a specific level of access you need to:
+
+- Create a role that allows being assumed by a web identity:
+```
+module "iam_assumable_role_my_role_name" {
+  source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
+  version                       = "~> v2.6.0"
+  create_role                   = true
+  role_name                     = "my-role-name"
+  provider_url                  = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "")
+  role_policy_arns              = [aws_iam_policy.external_dns.arn]
+  oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:my-service-account-name"]
+}
+```
+- Create a service account for your kubernetes service to use, with an annotation specifying which IAM role is associated:
+```
+resource "kubernetes_service_account" "my_service_account" {
+  metadata {
+    name        = "my-service-account-name"
+    namespace   = "kube-system"
+    annotations = {
+      "eks.amazonaws.com/role-arn" = module.iam_assumable_role.my_role_name.this_iam_role_arn
+    }
+  }
+}
+```
+- Use this service account in your deployment spec.
+
+Any pods that come up in that deployment will automatically have env vars injected called `AWS_ROLE_ARN` and `AWS_WEB_IDENTITY_TOKEN_FILE` that will let them use the AWS API.
+
+
 
 ## Organization
 

diff --git a/kubernetes/terraform/modules/kubernetes/cert_manager.tf b/kubernetes/terraform/modules/kubernetes/cert_manager.tf
@@ -0,0 +1,104 @@
+locals {
+  cert_manager_namespace  = "kube-system"
+  cert_manager_version    = "0.14.2"
+}
+
+# Reference an existing route53 zone
+data "aws_route53_zone" "public" {
+  name = var.external_dns_zone
+}
+
+# Cert-manager CRD manifest
+# https://github.com/jetstack/cert-manager/releases/download/v0.14.2/cert-manager.crds.yaml
+data "local_file" "cert_manager" {
+  filename = "${path.module}/files/cert-manager.crds.yaml"
+}
+
+# Install the cert manager Custom Resource Definitions (this can't be done via helm/terraform)
+resource "null_resource" "cert_manager" {
+  triggers = {
+    manifest_sha1 = "${sha1("${data.local_file.cert_manager.content}")}"
+  }
+  provisioner "local-exec" {
+    command = "kubectl apply --validate=false -f ${path.module}/files/cert-manager.crds.yaml"
+  }
+}
+
+data "helm_repository" "jetstack" {
+  name = "jetstack"
+  url  = "https://charts.jetstack.io"
+}
+
+resource "helm_release" "cert_manager" {
+  name       = "cert-manager"
+  repository = data.helm_repository.jetstack.metadata[0].name
+  chart      = "cert-manager"
+  version    = local.cert_manager_version
+  namespace  = local.cert_manager_namespace
+  set_string {
+    name  = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
+    value =  module.iam_assumable_role_cert_manager.this_iam_role_arn
+  }
+  set_string {
+    name  = "podAnnotations.eks\\.amazonaws\\.com/role-arn"
+    value =  module.iam_assumable_role_cert_manager.this_iam_role_arn
+  }
+  set {
+    name  = "securityContext.fsGroup"
+    value = "1001"
+  }
+}
+
+
+# Create a role using oidc to map service accounts
+module "iam_assumable_role_cert_manager" {
+  source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
+  version                       = "~> v2.6.0"
+  create_role                   = true
+  role_name                     = "<% .Name %>-k8s-${var.environment}-cert-manager"
+  provider_url                  = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "")
+  role_policy_arns              = [aws_iam_policy.cert_manager.arn]
+  oidc_fully_qualified_subjects = ["system:serviceaccount:${local.cert_manager_namespace}:cert-manager"]
+}
+
+resource "aws_iam_policy" "cert_manager" {
+  name_prefix = "cert-manager"
+  description = "EKS cert-manager policy for cluster ${var.cluster_name}"
+  policy      = data.aws_iam_policy_document.cert_manager_policy_doc.json
+}
+
+data "aws_iam_policy_document" "cert_manager_policy_doc" {
+  statement {
+    sid    = "ListZones"
+    effect = "Allow"
+
+    actions = [
+      "route53:ListHostedZonesByName"
+    ]
+
+    resources = ["*"]
+  }
+
+  statement {
+    sid    = "ReadWriteRecordsInZone"
+    effect = "Allow"
+
+    actions = [
+      "route53:ChangeResourceRecordSets",
+      "route53:ListResourceRecordSets"
+    ]
+
+    resources = ["arn:aws:route53:::hostedzone/${data.aws_route53_zone.public.zone_id}"]
+  }
+
+  statement {
+    sid    = "GetChange"
+    effect = "Allow"
+
+    actions = [
+      "route53:GetChange"
+    ]
+
+    resources = ["arn:aws:route53:::change/*"]
+  }
+}
diff --git a/kubernetes/terraform/modules/kubernetes/cluster_autoscaler.tf b/kubernetes/terraform/modules/kubernetes/cluster_autoscaler.tf
@@ -0,0 +1,92 @@
+locals {
+  cluster_autoscaler_namespace = "kube-system"
+}
+
+
+resource "helm_release" "cluster_autoscaler" {
+  name       = "cluster-autoscaler"
+  repository = data.helm_repository.stable.metadata[0].name
+  chart      = "cluster-autoscaler"
+  namespace  = local.cluster_autoscaler_namespace
+
+  set {
+    name  = "autoDiscovery.clusterName"
+    value = var.cluster_name
+  }
+  set {
+    name  = "rbac.create"
+    value = true
+  }
+  set_string {
+    name  = "rbac.serviceAccountAnnotations.eks\\.amazonaws\\.com/role-arn"
+    value =  module.iam_assumable_role_cluster_autoscaler.this_iam_role_arn
+  }
+  set {
+    name  = "awsRegion"
+    value = var.region
+  }
+  set {
+    name  = "extraArgs.v"
+    value = 2
+  }
+}
+
+
+# Create a role using oidc to map service accounts
+module "iam_assumable_role_cluster_autoscaler" {
+  source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
+  version                       = "~> v2.6.0"
+  create_role                   = true
+  role_name                     = "<% .Name %>-k8s-${var.environment}-cluster-autoscaler"
+  provider_url                  = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "")
+  role_policy_arns              = [aws_iam_policy.cluster_autoscaler.arn]
+  oidc_fully_qualified_subjects = ["system:serviceaccount:${local.cluster_autoscaler_namespace}:cluster-autoscaler-aws-cluster-autoscaler"]
+}
+
+resource "aws_iam_policy" "cluster_autoscaler" {
+  name_prefix = "cluster-autoscaler"
+  description = "EKS cluster-autoscaler policy for cluster ${var.cluster_name}"
+  policy      = data.aws_iam_policy_document.cluster_autoscaler_policy_doc.json
+}
+
+data "aws_iam_policy_document" "cluster_autoscaler_policy_doc" {
+  statement {
+    sid    = "eksWorkerAutoscalingAll"
+    effect = "Allow"
+
+    actions = [
+      "autoscaling:DescribeAutoScalingGroups",
+      "autoscaling:DescribeAutoScalingInstances",
+      "autoscaling:DescribeLaunchConfigurations",
+      "autoscaling:DescribeTags",
+      "ec2:DescribeLaunchTemplateVersions",
+    ]
+
+    resources = ["*"]
+  }
+
+  statement {
+    sid    = "eksWorkerAutoscalingOwn"
+    effect = "Allow"
+
+    actions = [
+      "autoscaling:SetDesiredCapacity",
+      "autoscaling:TerminateInstanceInAutoScalingGroup",
+      "autoscaling:UpdateAutoScalingGroup",
+    ]
+
+    resources = ["*"]
+
+    condition {
+      test     = "StringEquals"
+      variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${var.cluster_name}"
+      values   = ["owned"]
+    }
+
+    condition {
+      test     = "StringEquals"
+      variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled"
+      values   = ["true"]
+    }
+  }
+}
diff --git a/kubernetes/terraform/modules/kubernetes/external_dns.tf b/kubernetes/terraform/modules/kubernetes/external_dns.tf
@@ -1,5 +1,5 @@
 # Create a role using oidc to map service accounts
-module "iam_assumable_role" {
+module "iam_assumable_role_external_dns" {
   source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
   version                       = "~> v2.6.0"
   create_role                   = true
@@ -43,7 +43,7 @@ resource "kubernetes_service_account" "external_dns" {
     name        = "external-dns"
     namespace   = "kube-system"
     annotations = {
-      "eks.amazonaws.com/role-arn" = module.iam_assumable_role.this_iam_role_arn
+      "eks.amazonaws.com/role-arn" = module.iam_assumable_role_external_dns.this_iam_role_arn
     }
   }
 }
@@ -102,9 +102,6 @@ resource "kubernetes_deployment" "external_dns" {
         labels = {
           "app"    = "external-dns",
         }
-        annotations = {
-          "eks.amazonaws.com/role-arn" = module.iam_assumable_role.this_iam_role_arn
-        }
       }
       spec {
         container {