Merge pull request #3923 from cclerget/issue-3922

Set temporary single CPU affinity before cgroup cpuset transition.

docs/isolated-cpu-affinity-transition.md

@@ -0,0 +1,125 @@
+## Isolated CPU affinity transition
+
+The introduction of the kernel commit 46a87b3851f0d6eb05e6d83d5c5a30df0eca8f76
+in 5.7 has affected a deterministic scheduling behavior by distributing tasks
+across CPU cores within a cgroups cpuset. It means that `runc exec` might be
+impacted under some circumstances, by example when a container has been
+created within a cgroup cpuset entirely composed of isolated CPU cores
+usually sets either with `nohz_full` and/or `isolcpus` kernel boot parameters.
+
+Some containerized real-time applications are relying on this deterministic
+behavior and uses the first CPU core to run a slow thread while other CPU
+cores are fully used by the real-time threads with SCHED_FIFO policy.
+Such applications can prevent runc process from joining a container when the
+runc process is randomly scheduled on a CPU core owned by a real-time thread.
+
+Runc introduces a way to restore this behavior by adding the following
+annotation to the container runtime spec (`config.json`):
+
+`org.opencontainers.runc.exec.isolated-cpu-affinity-transition`
+
+This annotation can take one of those values:
+
+* `temporary` to temporarily set the runc process CPU affinity to the first
+isolated CPU core of the container cgroup cpuset.
+* `definitive`: to definitively set the runc process CPU affinity to the first
+isolated CPU core of the container cgroup cpuset.
+
+For example:
+
+```json
+ "annotations": {
+ "org.opencontainers.runc.exec.isolated-cpu-affinity-transition": "temporary"
+ }
+```
+
+__WARNING:__ `definitive` requires a kernel >= 6.2, also works with RHEL 9 and
+above.
+
+### How it works?
+
+When enabled and during `runc exec`, runc is looking for the `nohz_full` kernel
+boot parameter value and considers the CPUs in the list as isolated, it doesn't
+look for `isolcpus` boot parameter, it just assumes that `isolcpus` value is
+identical to `nohz_full` when specified. If `nohz_full` parameter is not found,
+runc also attempts to read the list from `/sys/devices/system/cpu/nohz_full`.
+
+Once it gets the isolated CPU list, it returns an eligible CPU core within the
+container cgroup cpuset based on those heuristics:
+
+* when there is not cpuset cores: no eligible CPU
+* when there is not isolated cores: no eligible CPU
+* when cpuset cores are not in isolated core list: no eligible CPU
+* when cpuset cores are all isolated cores: return the first CPU of the cpuset
+* when cpuset cores are mixed between housekeeping/isolated cores: return the
+ first housekeeping CPU not in isolated CPUs.
+
+The returned CPU core is then used to set the `runc init` CPU affinity before
+the container cgroup cpuset transition.
+
+#### Transition example
+
+`nohz_full` has the isolated cores `4-7`. A container has been created with
+the cgroup cpuset `4-7` to only run on the isolated CPU cores 4 to 7.
+`runc exec` is called by a process with CPU affinity set to `0-3`
+
+* with `temporary` transition:
+
+ runc exec (affinity 0-3) -> runc init (affinity 4) -> container process (affinity 4-7)
+
+* with `definitive` transition:
+
+ runc exec (affinity 0-3) -> runc init (affinity 4) -> container process (affinity 4)
+
+The difference between `temporary` and `definitive` is the container process
+affinity, `definitive` will constraint the container process to run on the
+first isolated CPU core of the cgroup cpuset, while `temporary` restore the
+CPU affinity to match the container cgroup cpuset.
+
+`definitive` transition might be helpful when `nohz_full` is used without
+`isolcpus` to avoid runc and container process to be a noisy neighbour for
+real-time applications.
+
+### How to use it with Kubernetes?
+
+Kubernetes doesn't manage container directly, instead it uses the Container Runtime
+Interface (CRI) to communicate with a software implementing this interface and responsible
+to manage the lifecycle of containers. There are popular CRI implementations like Containerd
+and CRI-O. Those implementations allows to pass pod annotations to the container runtime
+via the container runtime spec. Currently runc is the runtime used by default for both.
+
+#### Containerd configuration
+
+Containerd CRI uses runc by default but requires an extra step to pass the annotation to runc.
+You have to whitelist `org.opencontainers.runc.exec.isolated-cpu-affinity-transition` as a pod
+annotation allowed to be passed to the container runtime in `/etc/containerd/config.toml`:
+
+```toml
+[plugins."io.containerd.grpc.v1.cri".containerd]
+ default_runtime_name = "runc"
+ [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+ [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+ runtime_type = "io.containerd.runc.v2"
+ base_runtime_spec = "/etc/containerd/cri-base.json"
+ pod_annotations = ["org.opencontainers.runc.exec.isolated-cpu-affinity-transition"]
+```
+
+#### CRI-O configuration
+
+CRI-O doesn't require any extra step, however some annotations could be excluded by
+configuration.
+
+#### Pod deployment example
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: demo-pod
+ annotations:
+ org.opencontainers.runc.exec.isolated-cpu-affinity-transition: "temporary"
+spec:
+ containers:
+ - name: demo
+ image: registry.com/demo:latest
+```

features.go

@@ -68,6 +68,7 @@ var featuresCommand = cli.Command{
  "bundle",
  "org.systemd.property.", // prefix form
  "org.criu.config",
+ "org.opencontainers.runc.exec.isolated-cpu-affinity-transition",
  },
  }
 

libcontainer/cgroups/cgroups.go

@@ -71,4 +71,8 @@ type Manager interface {
 
  // OOMKillCount reports OOM kill count for the cgroup.
  OOMKillCount() (uint64, error)
+
+ // GetEffectiveCPUs returns the effective CPUs of the cgroup, an empty
+ // value means that the cgroups cpuset subsystem/controller is not enabled.
+ GetEffectiveCPUs() string
 }

libcontainer/cgroups/fs/fs.go

@@ -4,6 +4,8 @@ import (
  "errors"
  "fmt"
  "os"
+ "path/filepath"
+ "strings"
  "sync"
 
  "golang.org/x/sys/unix"
@@ -263,3 +265,28 @@ func (m *Manager) OOMKillCount() (uint64, error) {
 
  return c, err
 }
+
+func (m *Manager) GetEffectiveCPUs() string {
+ return GetEffectiveCPUs(m.Path("cpuset"), m.cgroups)
+}
+
+func GetEffectiveCPUs(cpusetPath string, cgroups *configs.Cgroup) string {
+ // Fast path.
+ if cgroups.CpusetCpus != "" {
+ return cgroups.CpusetCpus
+ } else if !strings.HasPrefix(cpusetPath, defaultCgroupRoot) {
+ return ""
+ }
+
+ // Iterates until it goes to the cgroup root path.
+ // It's required for containers in which cpuset controller
+ // is not enabled, in this case a parent cgroup is used.
+ for path := cpusetPath; path != defaultCgroupRoot; path = filepath.Dir(path) {
+ cpus, err := fscommon.GetCgroupParamString(path, "cpuset.effective_cpus")
+ if err == nil {
+ return cpus
+ }
+ }
+
+ return ""
+}

libcontainer/cgroups/fs2/fs2.go

@@ -4,11 +4,13 @@ import (
  "errors"
  "fmt"
  "os"
+ "path/filepath"
  "strings"
 
  "github.com/opencontainers/runc/libcontainer/cgroups"
  "github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
  "github.com/opencontainers/runc/libcontainer/configs"
+ "github.com/opencontainers/runc/libcontainer/utils"
 )
 
 type parseError = fscommon.ParseError
@@ -32,6 +34,9 @@ func NewManager(config *configs.Cgroup, dirPath string) (*Manager, error) {
  if err != nil {
  return nil, err
  }
+ } else {
+ // Clean path for safety.
+ dirPath = utils.CleanPath(dirPath)
  }
 
  m := &Manager{
@@ -316,3 +321,26 @@ func CheckMemoryUsage(dirPath string, r *configs.Resources) error {
 
  return nil
 }
+
+func (m *Manager) GetEffectiveCPUs() string {
+ // Fast path.
+ if m.config.CpusetCpus != "" {
+ return m.config.CpusetCpus
+ } else if !strings.HasPrefix(m.dirPath, UnifiedMountpoint) {
+ return ""
+ }
+
+ // Iterates until it goes outside of the cgroup root path.
+ // It's required for containers in which cpuset controller
+ // is not enabled, in this case a parent cgroup is used.
+ outsidePath := filepath.Dir(UnifiedMountpoint)
+
+ for path := m.dirPath; path != outsidePath; path = filepath.Dir(path) {
+ cpus, err := fscommon.GetCgroupParamString(path, "cpuset.cpus.effective")
+ if err == nil {
+ return cpus
+ }
+ }
+
+ return ""
+}

libcontainer/cgroups/systemd/v1.go

@@ -411,3 +411,7 @@ func (m *LegacyManager) Exists() bool {
 func (m *LegacyManager) OOMKillCount() (uint64, error) {
  return fs.OOMKillCount(m.Path("memory"))
 }
+
+func (m *LegacyManager) GetEffectiveCPUs() string {
+ return fs.GetEffectiveCPUs(m.Path("cpuset"), m.cgroups)
+}

libcontainer/cgroups/systemd/v2.go

@@ -514,3 +514,7 @@ func (m *UnifiedManager) Exists() bool {
 func (m *UnifiedManager) OOMKillCount() (uint64, error) {
  return m.fsMgr.OOMKillCount()
 }
+
+func (m *UnifiedManager) GetEffectiveCPUs() string {
+ return m.fsMgr.GetEffectiveCPUs()
+}

libcontainer/container_linux_test.go

@@ -69,6 +69,10 @@ func (m *mockCgroupManager) GetFreezerState() (configs.FreezerState, error) {
  return configs.Thawed, nil
 }
 
+func (m *mockCgroupManager) GetEffectiveCPUs() string {
+ return ""
+}
+
 type mockProcess struct {
  _pid int
  started uint64