ContainerCluster updates every 10 minutes

[pmialon]

Checklist

• I did not find a related open issue.
• I did not find a solution in the troubleshooting guide: (https://cloud.google.com/config-connector/docs/troubleshooting)
• If this issue is time-sensitive, I have submitted a corresponding issue with GCP support.

Bug Description

ContainerClusters are updated every 10 minutes with the annotation: cnrm.cloud.google.com/state-into-spec: absent

We use Config-Controller.

Additional Diagnostic Information

The output of gcloud container clusters describe my-cluster --location europe-west1 | yq '.monitoringConfig'

advancedDatapathObservabilityConfig:
  enableMetrics: true
  enableRelay: false
componentConfig:
  enableComponents:
    - APISERVER
    - CONTROLLER_MANAGER
    - SCHEDULER
    - SYSTEM_COMPONENTS
    - STORAGE
    - HPA
    - POD
    - DAEMONSET
    - DEPLOYMENT
    - STATEFULSET
managedPrometheusConfig:
  enabled: true

Kubernetes Cluster Version

v1.27.11-gke.1062001

Config Connector Version

1.116.0

Config Connector Mode

cluster mode

Log Output

With the following filter

resource.type="gke_cluster"
protoPayload.methodName="google.container.v1beta1.ClusterManager.UpdateCluster"

{
  "protoPayload": {
    "@type": "type.googleapis.com/google.cloud.audit.AuditLog",
    "authenticationInfo": {
      "principalEmail": "[email protected]",
      "principalSubject": "serviceAccount:[email protected]"
    },
    "requestMetadata": {
      "callerIp": "gce-internal-ip",
      "callerSuppliedUserAgent": "google-api-go-client/0.5 Terraform/ (+https://www.terraform.io) Terraform-Plugin-SDK/2.10.1 terraform-provider-google-beta/kcc/controller-manager,gzip(gfe)",
      "requestAttributes": {
        "time": "2024-05-14T08:23:35.684333905Z",
        "auth": {}
      },
      "destinationAttributes": {}
    },
    "serviceName": "container.googleapis.com",
    "methodName": "google.container.v1beta1.ClusterManager.UpdateCluster",
    "authorizationInfo": [
      {
        "resource": "projects/hidden/locations/europe-west1/clusters/hidden",
        "permission": "container.clusters.update",
        "granted": true,
        "resourceAttributes": {
          "service": "container.googleapis.com",
          "name": "projects/hidden/locations/europe-west1/clusters/hidden",
          "type": "container.googleapis.com/Cluster"
        },
        "permissionType": "ADMIN_WRITE"
      }
    ],
    "resourceName": "projects/hidden/locations/europe-west1/clusters/hidden",
    "request": {
      "update": {
        "desiredMonitoringConfig": {
          "advancedDatapathObservabilityConfig": {},
          "componentConfig": {
            "enableComponents": [
              "APISERVER",
              "CONTROLLER_MANAGER",
              "SCHEDULER",
              "SYSTEM_COMPONENTS",
              "STORAGE",
              "HPA",
              "POD",
              "DAEMONSET",
              "DEPLOYMENT",
              "STATEFULSET"
            ]
          },
          "managedPrometheusConfig": {
            "enabled": true
          }
        }
      },
      "name": "projects/hidden/locations/europe-west1/clusters/hidden",
      "@type": "type.googleapis.com/google.container.v1alpha1.UpdateClusterRequest"
    },
    "response": {
      "operationType": "UPDATE_CLUSTER",
      "status": "RUNNING",
      "selfLink": "https://container.googleapis.com/v1alpha1/projects/hidden/locations/europe-west1/operations/operation-1715675016040-6f41dd89-4d48-47a4-9a0f-47d9efe007bc",
      "targetLink": "https://container.googleapis.com/v1alpha1/projects/hidden/locations/europe-west1/clusters/hidden",
      "startTime": "2024-05-14T08:23:36.040101953Z",
      "@type": "type.googleapis.com/google.container.v1alpha1.Operation",
      "name": "operation-1715675016040-6f41dd89-4d48-47a4-9a0f-47d9efe007bc"
    },
    "resourceLocation": {
      "currentLocations": [
        "europe-west1"
      ]
    },
    "policyViolationInfo": {
      "orgPolicyViolationInfo": {}
    }
  },
  "insertId": "ldp2uie6dgwk",
  "resource": {
    "type": "gke_cluster",
    "labels": {
      "project_id": "hidden",
      "cluster_name": "hidden",
      "location": "europe-west1"
    }
  },
  "timestamp": "2024-05-14T08:23:36.182759565Z",
  "severity": "NOTICE",
  "logName": "projects/hidden/logs/cloudaudit.googleapis.com%2Factivity",
  "operation": {
    "id": "operation-1715675016040-6f41dd89-4d48-47a4-9a0f-47d9efe007bc",
    "producer": "container.googleapis.com",
    "first": true
  },
  "receiveTimestamp": "2024-05-14T08:23:37.018773004Z"
}

Steps to reproduce the issue

Create a ContainerCluster with the provided yaml.

YAML snippets

apiVersion: container.cnrm.cloud.google.com/v1beta1
kind: ContainerCluster
metadata:
  annotations:
    cnrm.cloud.google.com/remove-default-node-pool: "false"
    cnrm.cloud.google.com/state-into-spec: absent
  labels:
    mesh_id: proj-000000000000
  name: my-cluster
  namespace: config-control
spec:
  addonsConfig:
    dnsCacheConfig:
      enabled: true
    gcePersistentDiskCsiDriverConfig:
      enabled: true
    httpLoadBalancing:
      disabled: false
  authenticatorGroupsConfig:
    securityGroup: [email protected]
  clusterAutoscaling:
    autoProvisioningDefaults:
      bootDiskKMSKeyRef:
        name: my-cluster-gke-boot-disk
      oauthScopes:
      - https://www.googleapis.com/auth/cloud-platform
      serviceAccountRef:
        name: my-cluster-nodes
  databaseEncryption:
    keyName: projects/my-cluster/locations/europe-west1/keyRings/gke/cryptoKeys/gke-database
    state: ENCRYPTED
  datapathProvider: ADVANCED_DATAPATH
  enableAutopilot: true
  enableShieldedNodes: true
  initialNodeCount: 1
  ipAllocationPolicy:
    clusterSecondaryRangeName: my-cluster-pods
    servicesSecondaryRangeName: my-cluster-svc
  location: europe-west1
  loggingConfig:
    enableComponents:
    - SYSTEM_COMPONENTS
    - WORKLOADS
  masterAuthorizedNetworksConfig:
    cidrBlocks:
    - cidrBlock: 172.16.0.0/13
  minMasterVersion: 1.28.3-gke.1286000
  monitoringConfig:
    advancedDatapathObservabilityConfig:
    - enableMetrics: true
      relayMode: DISABLED
    enableComponents:
    - APISERVER
    - CONTROLLER_MANAGER
    - SCHEDULER
    - SYSTEM_COMPONENTS
    - STORAGE
    - HPA
    - POD
    - DAEMONSET
    - DEPLOYMENT
    - STATEFULSET
    managedPrometheus:
      enabled: true
  networkRef:
    name: sharedvpc-network
  nodePoolAutoConfig:
    networkTags:
      tags:
      - my-cluster
      - asm-multi-cluster
  privateClusterConfig:
    enablePrivateEndpoint: false
    enablePrivateNodes: true
    masterGlobalAccessConfig:
      enabled: true
    masterIpv4CidrBlock: 172.25.32.32/28
  releaseChannel:
    channel: STABLE
  securityPostureConfig:
    mode: BASIC
    vulnerabilityMode: VULNERABILITY_ENTERPRISE
  subnetworkRef:
    name: my-cluster-gke
  workloadIdentityConfig:
    workloadPool: my-cluster.svc.id.goog