feat: Add support for Pod Identity assocation on Karpenter sub-module

examples/karpenter/README.md

@@ -22,10 +22,40 @@ aws eks --region eu-west-1 update-kubeconfig --name ex-karpenter
 kubectl scale deployment inflate --replicas 5
 
 # You can watch Karpenter's controller logs with
-kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller
+kubectl logs -f -n kube-system -l app.kubernetes.io/name=karpenter -c controller
 ```
 
-You should see a new node named `karpenter.sh/provisioner-name/default` eventually come up in the console; this was provisioned by Karpenter in response to the scaled deployment above.
+Validate if the Amazon EKS Addons Pods are running in the Managed Node Group and the `inflate` application Pods are running on Karpenter provisioned Nodes.
+
+```bash
+kubectl get nodes -L karpenter.sh/registered
+NAME STATUS ROLES AGE VERSION REGISTERED
+ip-10-0-16-155.eu-west-1.compute.internal Ready <none> 100s v1.29.3-eks-ae9a62a true
+ip-10-0-3-23.eu-west-1.compute.internal Ready <none> 6m1s v1.29.3-eks-ae9a62a
+ip-10-0-41-2.eu-west-1.compute.internal Ready <none> 6m3s v1.29.3-eks-ae9a62a
+
+kubectl get pods -A -o custom-columns=NAME:.metadata.name,NODE:.spec.nodeName
+
+NAME NODE
+inflate-75d744d4c6-nqwz8 ip-10-0-16-155.eu-west-1.compute.internal
+inflate-75d744d4c6-nrqnn ip-10-0-16-155.eu-west-1.compute.internal
+inflate-75d744d4c6-sp4dx ip-10-0-16-155.eu-west-1.compute.internal
+inflate-75d744d4c6-xqzd9 ip-10-0-16-155.eu-west-1.compute.internal
+inflate-75d744d4c6-xr6p5 ip-10-0-16-155.eu-west-1.compute.internal
+aws-node-mnn7r ip-10-0-3-23.eu-west-1.compute.internal
+aws-node-rkmvm ip-10-0-16-155.eu-west-1.compute.internal
+aws-node-s4slh ip-10-0-41-2.eu-west-1.compute.internal
+coredns-68bd859788-7rcfq ip-10-0-3-23.eu-west-1.compute.internal
+coredns-68bd859788-l78hw ip-10-0-41-2.eu-west-1.compute.internal
+eks-pod-identity-agent-gbx8l ip-10-0-41-2.eu-west-1.compute.internal
+eks-pod-identity-agent-s7vt7 ip-10-0-16-155.eu-west-1.compute.internal
+eks-pod-identity-agent-xwgqw ip-10-0-3-23.eu-west-1.compute.internal
+karpenter-79f59bdfdc-9q5ff ip-10-0-41-2.eu-west-1.compute.internal
+karpenter-79f59bdfdc-cxvhr ip-10-0-3-23.eu-west-1.compute.internal
+kube-proxy-7crbl ip-10-0-41-2.eu-west-1.compute.internal
+kube-proxy-jtzds ip-10-0-16-155.eu-west-1.compute.internal
+kube-proxy-sm42c ip-10-0-3-23.eu-west-1.compute.internal
+```
 
 ### Tear Down & Clean-Up
 

examples/karpenter/main.tf

@@ -71,53 +71,40 @@ module "eks" {
  enable_cluster_creator_admin_permissions = true
 
  cluster_addons = {
- coredns = {
- configuration_values = jsonencode({
- computeType = "Fargate"
- # Ensure that we fully utilize the minimum amount of resources that are supplied by
- # Fargate https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html
- # Fargate adds 256 MB to each pod's memory reservation for the required Kubernetes
- # components (kubelet, kube-proxy, and containerd). Fargate rounds up to the following
- # compute configuration that most closely matches the sum of vCPU and memory requests in
- # order to ensure pods always have the resources that they need to run.
- resources = {
- limits = {
- cpu = "0.25"
- # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the
- # request/limit to ensure we can fit within that task
- memory = "256M"
- }
- requests = {
- cpu = "0.25"
- # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the
- # request/limit to ensure we can fit within that task
- memory = "256M"
- }
- }
- })
- }
- kube-proxy = {}
- vpc-cni = {}
+ coredns = {}
+ kube-proxy = {}
+ vpc-cni = {}
+ eks-pod-identity-agent = {}
  }
 
  vpc_id = module.vpc.vpc_id
  subnet_ids = module.vpc.private_subnets
  control_plane_subnet_ids = module.vpc.intra_subnets
 
- # Fargate profiles use the cluster primary security group so these are not utilized
- create_cluster_security_group = false
- create_node_security_group = false
-
- fargate_profiles = {
- karpenter = {
- selectors = [
- { namespace = "karpenter" }
- ]
- }
- kube-system = {
- selectors = [
- { namespace = "kube-system" }
- ]
+ # eks_managed_node_group_defaults = {
+ # ami_type = "BOTTLEROCKET_x86_64"
+ # instance_types = ["t3.large", "t3a.large"]
+
+ # iam_role_attach_cni_policy = true
+ # }
+
+ eks_managed_node_groups = {
+ initial = {
+ instance_types = ["m5.large"]
+
+ min_size = 1
+ max_size = 3
+ desired_size = 2
+
+ taints = {
+ # This Taint aims to keep just EKS Addons and Karpenter running on this MNG
+ # The pods that do not tolerate this taint should run on nodes created by Karpenter
+ addons = {
+ key = "CriticalAddonsOnly"
+ value = "true"
+ effect = "NO_SCHEDULE"
+ },
+ }
  }
  }
 
@@ -138,9 +125,8 @@ module "karpenter" {
 
  cluster_name = module.eks.cluster_name
 
- # EKS Fargate currently does not support Pod Identity
- enable_irsa = true
- irsa_oidc_provider_arn = module.eks.oidc_provider_arn
+ enable_pod_identity = true
+ create_pod_identity_association = true
 
  # Used to attach additional IAM policies to the Karpenter node IAM role
  node_iam_role_additional_policies = {
@@ -162,14 +148,14 @@ module "karpenter_disabled" {
 ################################################################################
 
 resource "helm_release" "karpenter" {
- namespace = "karpenter"
+ namespace = "kube-system"
  create_namespace = true
  name = "karpenter"
  repository = "oci://public.ecr.aws/karpenter"
  repository_username = data.aws_ecrpublic_authorization_token.token.user_name
  repository_password = data.aws_ecrpublic_authorization_token.token.password
  chart = "karpenter"
- version = "0.35.1"
+ version = "0.36.1"
  wait = false
 
  values = [
@@ -178,14 +164,6 @@ resource "helm_release" "karpenter" {
  clusterName: ${module.eks.cluster_name}
  clusterEndpoint: ${module.eks.cluster_endpoint}
  interruptionQueue: ${module.karpenter.queue_name}
- serviceAccount:
- annotations:
- eks.amazonaws.com/role-arn: ${module.karpenter.iam_role_arn}
- tolerations:
- - key: 'eks.amazonaws.com/compute-type'
- operator: Equal
- value: fargate
- effect: "NoSchedule"
  EOT
  ]
 }

modules/karpenter/README.md

@@ -11,6 +11,7 @@ In the following example, the Karpenter module will create:
 - A Node IAM role that Karpenter will use to create an Instance Profile for the nodes to receive IAM permissions
 - An access entry for the Node IAM role to allow nodes to join the cluster
 - SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc.
+- A Pod Identity association to grant Karpenter controller access provided by the IAM Role
 
 ```hcl
 module "eks" {
@@ -20,19 +21,19 @@ module "eks" {
 }
 
 module "karpenter" {
- source = "terraform-aws-modules/eks/aws//modules/karpenter"
+ source = "../../modules/karpenter"
 
  cluster_name = module.eks.cluster_name
 
- # Attach additional IAM policies to the Karpenter node IAM role
+ enable_pod_identity = true
+ create_pod_identity_association = true
+
+ # Used to attach additional IAM policies to the Karpenter node IAM role
  node_iam_role_additional_policies = {
  AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
  }
 
- tags = {
- Environment = "dev"
- Terraform = "true"
- }
+ tags = local.tags
 }
 ```
 
@@ -104,6 +105,7 @@ No modules.
 | [aws_cloudwatch_event_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
 | [aws_cloudwatch_event_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
 | [aws_eks_access_entry.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource |
+| [aws_eks_pod_identity_association.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_pod_identity_association) | resource |
 | [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource |
 | [aws_iam_policy.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_role.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
@@ -135,6 +137,7 @@ No modules.
 | <a name="input_create_iam_role"></a> [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created | `bool` | `true` | no |
 | <a name="input_create_instance_profile"></a> [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `false` | no |
 | <a name="input_create_node_iam_role"></a> [create\_node\_iam\_role](#input\_create\_node\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no |
+| <a name="input_create_pod_identity_association"></a> [create\_pod\_identity\_association](#input\_create\_pod\_identity\_association) | Determines whether to create pod identity association | `bool` | `false` | no |
 | <a name="input_enable_irsa"></a> [enable\_irsa](#input\_enable\_irsa) | Determines whether to enable support for IAM role for service accounts | `bool` | `false` | no |
 | <a name="input_enable_pod_identity"></a> [enable\_pod\_identity](#input\_enable\_pod\_identity) | Determines whether to enable support for EKS pod identity | `bool` | `true` | no |
 | <a name="input_enable_spot_termination"></a> [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no |
@@ -153,6 +156,7 @@ No modules.
 | <a name="input_irsa_assume_role_condition_test"></a> [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no |
 | <a name="input_irsa_namespace_service_accounts"></a> [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` | <pre>[<br> "karpenter:karpenter"<br>]</pre> | no |
 | <a name="input_irsa_oidc_provider_arn"></a> [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no |
+| <a name="input_namespace"></a> [namespace](#input\_namespace) | Namespace where pod identity will be created | `string` | `"kube-system"` | no |
 | <a name="input_node_iam_role_additional_policies"></a> [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no |
 | <a name="input_node_iam_role_arn"></a> [node\_iam\_role\_arn](#input\_node\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no |
 | <a name="input_node_iam_role_attach_cni_policy"></a> [node\_iam\_role\_attach\_cni\_policy](#input\_node\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no |
@@ -168,6 +172,7 @@ No modules.
 | <a name="input_queue_managed_sse_enabled"></a> [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no |
 | <a name="input_queue_name"></a> [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no |
 | <a name="input_rule_name_prefix"></a> [rule\_name\_prefix](#input\_rule\_name\_prefix) | Prefix used for all event bridge rules | `string` | `"Karpenter"` | no |
+| <a name="input_service_account"></a> [service\_account](#input\_service\_account) | Service account where pod identity will be created | `string` | `"karpenter"` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no |
 
 ## Outputs
@@ -182,11 +187,13 @@ No modules.
 | <a name="output_instance_profile_id"></a> [instance\_profile\_id](#output\_instance\_profile\_id) | Instance profile's ID |
 | <a name="output_instance_profile_name"></a> [instance\_profile\_name](#output\_instance\_profile\_name) | Name of the instance profile |
 | <a name="output_instance_profile_unique"></a> [instance\_profile\_unique](#output\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile |
+| <a name="output_namespace"></a> [namespace](#output\_namespace) | Namespace for the Karpenter Pod Identity association |
 | <a name="output_node_access_entry_arn"></a> [node\_access\_entry\_arn](#output\_node\_access\_entry\_arn) | Amazon Resource Name (ARN) of the node Access Entry |
 | <a name="output_node_iam_role_arn"></a> [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role |
 | <a name="output_node_iam_role_name"></a> [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | The name of the node IAM role |
 | <a name="output_node_iam_role_unique_id"></a> [node\_iam\_role\_unique\_id](#output\_node\_iam\_role\_unique\_id) | Stable and unique string identifying the node IAM role |
 | <a name="output_queue_arn"></a> [queue\_arn](#output\_queue\_arn) | The ARN of the SQS queue |
 | <a name="output_queue_name"></a> [queue\_name](#output\_queue\_name) | The name of the created Amazon SQS queue |
 | <a name="output_queue_url"></a> [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue |
+| <a name="output_service_account"></a> [service\_account](#output\_service\_account) | Service Account to be associated with Karpenter Pod Identity |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/karpenter/main.tf

@@ -411,6 +411,18 @@ resource "aws_iam_role_policy_attachment" "controller_additional" {
  policy_arn = each.value
 }
 
+# Pod Identity Association
+
+resource "aws_eks_pod_identity_association" "karpenter" {
+ # TODO Update this condition in the next breaking change
+ count = local.create_iam_role && var.enable_pod_identity && var.create_pod_identity_association ? 1 : 0
+
+ cluster_name = var.cluster_name
+ namespace = var.namespace
+ service_account = var.service_account
+ role_arn = aws_iam_role.controller[0].arn
+}
+
 ################################################################################
 # Node Termination Queue
 ################################################################################

modules/karpenter/outputs.tf

@@ -96,3 +96,17 @@ output "instance_profile_unique" {
  description = "Stable and unique string identifying the IAM instance profile"
  value = try(aws_iam_instance_profile.this[0].unique_id, null)
 }
+
+################################################################################
+# Pod Identity
+################################################################################
+
+output "namespace" {
+ description = "Namespace for the Karpenter Pod Identity association"
+ value = var.namespace
+}
+
+output "service_account" {
+ description = "Service Account to be associated with Karpenter Pod Identity"
+ value = var.service_account
+}

modules/karpenter/variables.tf

@@ -138,6 +138,28 @@ variable "irsa_assume_role_condition_test" {
  default = "StringEquals"
 }
 
+################################################################################
+# Pod Identity
+################################################################################
+# TODO: Remove this in the next breaking change
+variable "create_pod_identity_association" {
+ description = "Determines whether to create pod identity association"
+ type = bool
+ default = false
+}
+
+variable "namespace" {
+ description = "Namespace where pod identity will be created"
+ type = string
+ default = "kube-system"
+}
+
+variable "service_account" {
+ description = "Service account where pod identity will be created"
+ type = string
+ default = "karpenter"
+}
+
 ################################################################################
 # Node Termination Queue
 ################################################################################