cluster-autoscaler-vanilla.yaml

---
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-addon: cluster-autoscaler.addons.k8s.io
    k8s-app: cluster-autoscaler
  name: cluster-autoscaler
  namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cluster-autoscaler
  labels:
    k8s-addon: cluster-autoscaler.addons.k8s.io
    k8s-app: cluster-autoscaler
rules:
  - verbs:
      - create
      - patch
    apiGroups:
      - ""
    resources:
      - events
      - endpoints
  - verbs:
      - create
    apiGroups:
      - ""
    resources:
      - pods/eviction
  - verbs:
      - update
    apiGroups:
      - ""
    resources:
      - nodes/status
  - verbs:
      - update
    apiGroups:
      - ""
    resources:
      - pods/status
  - verbs:
      - get
      - update
      - delete
    apiGroups:
      - ""
    resources:
      - endpoints
    resourceNames:
      - cluster-autoscaler
      - vsphere-autoscaler
  - verbs:
      - watch
      - list
      - get
      - update
      - delete
      - patch
    apiGroups:
      - ""
    resources:
      - nodes
  - verbs:
      - watch
      - list
      - get
      - update
    apiGroups:
      - ""
    resources:
      - pods
      - namespaces
      - services
      - replicationcontrollers
      - persistentvolumeclaims
      - persistentvolumes
  - verbs:
      - watch
      - list
      - get
    apiGroups:
      - extensions
    resources:
      - replicasets
      - daemonsets
  - verbs:
      - watch
      - list
    apiGroups:
      - policy
    resources:
      - poddisruptionbudgets
  - verbs:
      - watch
      - list
      - get
      - delete
    apiGroups:
      - apps
    resources:
      - statefulsets
      - replicasets
      - daemonsets
  - verbs:
      - get
      - list
      - watch
    apiGroups:
      - storage.k8s.io
    resources:
      - storageclasses
      - csinodes
  - verbs:
      - watch
      - list
      - get
    apiGroups:
      - batch
    resources:
      - jobs
      - cronjobs
  - verbs:
      - create
    apiGroups:
      - coordination.k8s.io
    resources:
      - leases
  - verbs:
      - get
      - update
    apiGroups:
      - coordination.k8s.io
    resources:
      - leases
    resourceNames:
      - cluster-autoscaler
  - verbs:
      - list
      - get
      - update
      - watch
    apiGroups:
      - storage.k8s.io
    resources:
      - csistoragecapacities
      - csidrivers
  - verbs:
      - list
      - get
      - update
      - watch
      - delete
      - patch
    apiGroups:
      - nodemanager.aldunelabs.com
    resources:
      - "*"
  - verbs:
      - list
      - get
      - update
      - watch
      - delete
      - create
    apiGroups:
      - apiextensions.k8s.io
    resources:
      - customresourcedefinitions
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: cluster-autoscaler
  namespace: kube-system
  labels:
    k8s-addon: cluster-autoscaler.addons.k8s.io
    k8s-app: cluster-autoscaler
rules:
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["create"]
  - apiGroups: [""]
    resources: ["configmaps"]
    resourceNames: ["cluster-autoscaler-status"]
    verbs: ["delete", "get", "update"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: cluster-autoscaler
  labels:
    k8s-addon: cluster-autoscaler.addons.k8s.io
    k8s-app: cluster-autoscaler
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-autoscaler
subjects:
  - kind: ServiceAccount
    name: cluster-autoscaler
    namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: cluster-autoscaler
  namespace: kube-system
  labels:
    k8s-addon: cluster-autoscaler.addons.k8s.io
    k8s-app: cluster-autoscaler
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: cluster-autoscaler
subjects:
  - kind: ServiceAccount
    name: cluster-autoscaler
    namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: config-cluster-autoscaler
  namespace: kube-system
data:
  cloud-config: |-
    address: unix:/var/run/cluster-autoscaler/vmware.sock
    key: "/etc/ssl/client-cert/tls.key"
    cert: "/etc/ssl/client-cert/tls.crt"
    cacert: "/etc/ssl/client-cert/ca.crt"
  kubernetes-vmware-autoscaler.json: |
    {
      "distribution": "external",
      "use-external-etcd": false,
      "src-etcd-ssl-dir": "/etc/etcd/ssl",
      "dst-etcd-ssl-dir": "/etc/kubernetes/pki/etcd",
      "kubernetes-pki-srcdir": "/etc/kubernetes/pki",
      "kubernetes-pki-dstdir": "/etc/kubernetes/pki",
      "network": "unix",
      "listen": "/var/run/cluster-autoscaler/vmware.sock",
      "cert-private-key": "/etc/ssl/client-cert/tls.key",
      "cert-public-key": "/etc/ssl/client-cert/tls.crt",
      "cert-ca": "/etc/ssl/client-cert/ca.crt",
      "secret": "vmware",
      "minNode": 0,
      "maxNode": 9,
      "maxNode-per-cycle": 2,
      "node-name-prefix": "autoscaled",
      "managed-name-prefix": "managed",
      "controlplane-name-prefix": "master",
      "nodePrice": 0,
      "podPrice": 0,
      "image": "jammy-kubernetes-cni-flannel-v1.27.8-containerd-amd64",
      "optionals": {
        "pricing": false,
        "getAvailableMachineTypes": false,
        "newNodeGroup": false,
        "templateNodeInfo": false,
        "createNodeGroup": false,
        "deleteNodeGroup": false
      },
      "external":
        {
          "address": "192.168.1.120:6443",
          "token": "h1g55p.hm4rg52ymloax182",
          "join-command": "/usr/local/bin/join-cluster.sh"
          "config-path": "/etc/default/vmware-autoscaler-config.yaml"
          "extra-config": {
            "...": "..."
          }
      },
      "default-machine": "large",
      "machines": {
        "tiny": {
          "memsize": 2048,
          "vcpus": 2,
          "disksize": 10240
        },
        "small": {
          "memsize": 4096,
          "vcpus": 2,
          "disksize": 20480
        },
        "medium": {
          "memsize": 4096,
          "vcpus": 4,
          "disksize": 20480
        },
        "large": {
          "memsize": 8192,
          "vcpus": 4,
          "disksize": 51200
        },
        "xlarge": {
          "memsize": 16384,
          "vcpus": 4,
          "disksize": 102400
        },
        "2xlarge": {
          "memsize": 16384,
          "vcpus": 8,
          "disksize": 102400
        },
        "4xlarge": {
          "memsize": 32768,
          "vcpus": 8,
          "disksize": 102400
        }
      },
      "node-labels": [
        "topology.kubernetes.io/region=home",
        "topology.kubernetes.io/zone=office",
        "topology.csi.vmware.com/k8s-region=home",
        "topology.csi.vmware.com/k8s-zone=office"
      ],
      "cloud-init": {
        "package_update": false,
        "package_upgrade": false,
        "runcmd": [
          "echo 1 > /sys/block/sda/device/rescan",
          "growpart /dev/sda 1",
          "resize2fs /dev/sda1",
          "echo '192.168.1.120 vmware-ca-k8s-masterkube vmware-ca-k8s-masterkube.acme.com' >> /etc/hosts"
        ]
      },
      "ssh-infos": {
        "user": "kubernetes",
        "ssh-private-key": "/root/.ssh/id_rsa"
      },
      "autoscaling-options": {
        "scaleDownUtilizationThreshold": 0.5,
        "scaleDownGpuUtilizationThreshold": 0.5,
        "scaleDownUnneededTime": "1m",
        "scaleDownUnreadyTime": "1m",
      },
      "vmware": {
        "vmware-ca-k8s": {
          "url": "https://administrator@acme.com:mySecret@vsphere.acme.com/sdk",
          "uid": "administrator@vsphere.acme.com",
          "password": "mySecret",
          "insecure": true,
          "dc": "DC01",
          "datastore": "datastore1",
          "resource-pool": "ACME/Resources/FR",
          "vmFolder": "HOME",
          "timeout": 300,
          "template-name": "jammy-kubernetes-cni-flannel-v1.27.8-containerd-amd64",
          "template": false,
          "linked": false,
          "customization": "",
          "network": {
            "domain": "acme.com",
            "dns": {
              "search": [
                "acme.com"
              ],
              "nameserver": [
                "10.0.0.1"
              ]
            },
            "interfaces": [
              {
                "primary": false,
                "exists": true,
                "network": "VM Network",
                "adapter": "vmxnet3",
                "mac-address": "generate",
                "nic": "eth0",
                "dhcp": true,
                "use-dhcp-routes": true,
                "routes": [
                  {
                    "to": "172.30.0.0/16",
                    "via": "10.0.0.5",
                    "metric": 500
                  }
                ]
              },
              {
                "primary": true,
                "exists": true,
                "network": "VM Private",
                "adapter": "vmxnet3",
                "mac-address": "generate",
                "nic": "eth1",
                "dhcp": true,
                "use-dhcp-routes": false,
                "address": "192.168.1.124",
                "gateway": "10.0.0.1",
                "netmask": "255.255.255.0",
                "routes": []
              }
            ]
          }
        }
      }
    }
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: cluster-autoscaler
  namespace: kube-system
  labels:
    k8s-app: cluster-autoscaler
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: cluster-autoscaler
  template:
    metadata:
      labels:
        k8s-app: cluster-autoscaler
    spec:
      volumes:
        - name: cluster-socket
          emptyDir: {}
        - name: config-cluster-autoscaler
          configMap:
            name: config-cluster-autoscaler
            defaultMode: 420
        - name: ssl-certs
          hostPath:
            path: /etc/ssl/certs/ca-certificates.crt
        - name: autoscaler-ssh-keys
          secret:
            secretName: autoscaler-ssh-keys
            defaultMode: 384
        - name: etcd-ssl
          secret:
            secretName: etcd-ssl
            defaultMode: 384
        - name: kubernetes-pki
          configMap:
            name: kubernetes-pki
            defaultMode: 420
        - name: cluster-autoscaler-grpc-client-cert
          secret:
            secretName: cluster-autoscaler-grpc-client-cert
            defaultMode: 0400
      initContainers:
        - name: cluster-autoscaler-init
          image: busybox
          command:
            - /bin/sh
            - "-c"
            - rm -f /var/run/cluster-autoscaler/vmware.sock
          volumeMounts:
            - name: cluster-socket
              mountPath: /var/run/cluster-autoscaler
      containers:
        - name: vsphere-autoscaler
          image: fred78290/vsphere-autoscaler:v1.27.8
          command:
            - /usr/local/bin/vsphere-autoscaler
            - "--no-use-external-etcd"
            - "--src-etcd-ssl-dir=/etc/etcd/ssl"
            - "--dst-etcd-ssl-dir=/etc/etcd/ssl"
            - "--config=/etc/cluster/kubernetes-vmware-autoscaler.json"
            - "--save=/var/run/cluster-autoscaler/vmware-autoscaler-state.json"
            - "--log-level=info"
          resources:
            limits:
              cpu: 100m
              memory: 300Mi
            requests:
              cpu: 100m
              memory: 300Mi
          volumeMounts:
            - name: cluster-socket
              mountPath: /var/run/cluster-autoscaler
            - name: config-cluster-autoscaler
              mountPath: /etc/cluster
            - name: autoscaler-ssh-keys
              mountPath: /root/.ssh
            - name: etcd-ssl
              mountPath: /etc/etcd/ssl
            - name: kubernetes-pki
              mountPath: /etc/kubernetes/pki
            - name: cluster-autoscaler-grpc-client-cert
              mountPath: "/etc/ssl/client-cert"
        - name: cluster-autoscaler
          image: k8s.gcr.io/autoscaling/cluster-autoscaler:v1.27.8
          command:
            - ./cluster-autoscaler
            - "--v=1"
            - "--stderrthreshold=info"
            - "--cloud-provider=externalgrpc"
            - "--cloud-config=/etc/cluster/cloud-config"
            - "--max-nodes-total=9"
            - "--cores-total=0:16"
            - "--memory-total=0:48"
            - "--node-autoprovisioning-enabled"
            - "--max-autoprovisioned-node-group-count=1"
            - "--scale-down-enabled=true"
            - "--scale-down-delay-after-add=1m"
            - "--scale-down-delay-after-delete=1m"
            - "--scale-down-delay-after-failure=1m"
            - "--scale-down-unneeded-time=1m"
            - "--scale-down-unready-time=1m"
            - "--unremovable-node-recheck-timeout=1m"
          resources:
            limits:
              cpu: 100m
              memory: 300Mi
            requests:
              cpu: 100m
              memory: 300Mi
          volumeMounts:
            - name: cluster-socket
              mountPath: /var/run/cluster-autoscaler
            - name: ssl-certs
              readOnly: true
              mountPath: /etc/ssl/certs/ca-certificates.crt
            - name: config-cluster-autoscaler
              readOnly: true
              mountPath: /etc/cluster
            - name: cluster-autoscaler-grpc-client-cert
              mountPath: "/etc/ssl/client-cert"
      restartPolicy: Always
      dnsPolicy: ClusterFirst
      serviceAccountName: cluster-autoscaler
      serviceAccount: cluster-autoscaler
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
        - key: node-role.kubernetes.io/control-plane
          effect: NoSchedule