cri: implement RuntimeConfig rpc

The rpc only reports one field, i.e. the cgroup driver, to kubelet.
Containerd determines the effective cgroup driver by looking at all
runtime handlers, starting from the default runtime handler (the rest in
alphabetical order), and returning the cgroup driver setting of the
first runtime handler that supports one. If no runtime handler supports
cgroup driver (i.e. has a config option for it) containerd falls back to
auto-detection, returning systemd if systemd is running and cgroupfs
otherwise.

This patch implements the CRI server side of Kubernetes KEP-4033:
https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/4033-group-driver-detection-over-cri

Signed-off-by: Markus Lehtonen <[email protected]>

go.sum

@@ -887,6 +887,7 @@ github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
 github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
+github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=

pkg/cri/instrument/instrumented_service.go

@@ -621,3 +621,19 @@ func (in *instrumentedService) ListPodSandboxMetrics(ctx context.Context, r *run
 	res, err = in.c.ListPodSandboxMetrics(ctx, r)
 	return res, errdefs.ToGRPC(err)
 }
+
+func (in *instrumentedService) RuntimeConfig(ctx context.Context, r *runtime.RuntimeConfigRequest) (res *runtime.RuntimeConfigResponse, err error) {
+	if err := in.checkInitialized(); err != nil {
+		return nil, err
+	}
+	log.G(ctx).Tracef("RuntimeConfig")
+	defer func() {
+		if err != nil {
+			log.G(ctx).WithError(err).Error("RuntimeConfig failed")
+		} else {
+			log.G(ctx).Tracef("RuntimeConfig returns config %+v", res)
+		}
+	}()
+	res, err = in.c.RuntimeConfig(ctx, r)
+	return res, errdefs.ToGRPC(err)
+}

pkg/cri/sbserver/runtime_config.go

@@ -0,0 +1,31 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sbserver
+
+import (
+	"context"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+// RuntimeConfig returns configuration information of the runtime.
+func (c *criService) RuntimeConfig(ctx context.Context, r *runtime.RuntimeConfigRequest) (*runtime.RuntimeConfigResponse, error) {
+	resp := &runtime.RuntimeConfigResponse{
+		Linux: c.getLinuxRuntimeConfig(ctx),
+	}
+	return resp, nil
+}

pkg/cri/sbserver/runtime_config_linux.go

@@ -0,0 +1,81 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sbserver
+
+import (
+	"context"
+	"sort"
+
+	"github.com/containerd/containerd/log"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
+	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+func (c *criService) getLinuxRuntimeConfig(ctx context.Context) *runtime.LinuxRuntimeConfiguration {
+	return &runtime.LinuxRuntimeConfiguration{CgroupDriver: c.getCgroupDriver(ctx)}
+}
+
+func (c *criService) getCgroupDriver(ctx context.Context) runtime.CgroupDriver {
+	// Go through the runtime handlers in a predictable order, starting from the
+	// default handler, others sorted in alphabetical order
+	handlerNames := make([]string, 0, len(c.config.ContainerdConfig.Runtimes))
+	for n := range c.config.ContainerdConfig.Runtimes {
+		handlerNames = append(handlerNames, n)
+	}
+	sort.Slice(handlerNames, func(i, j int) bool {
+		if handlerNames[i] == c.config.ContainerdConfig.DefaultRuntimeName {
+			return true
+		}
+		if handlerNames[j] == c.config.ContainerdConfig.DefaultRuntimeName {
+			return false
+		}
+		return handlerNames[i] < handlerNames[j]
+	})
+
+	for _, handler := range handlerNames {
+		opts, err := generateRuntimeOptions(c.config.ContainerdConfig.Runtimes[handler])
+		if err != nil {
+			log.G(ctx).Debugf("failed to parse runtime handler options for %q", handler)
+			continue
+		}
+		if d, ok := getCgroupDriverFromRuntimeHandlerOpts(opts); ok {
+			return d
+		}
+		log.G(ctx).Debugf("runtime handler %q does not provide cgroup driver information", handler)
+	}
+
+	// If no runtime handlers have a setting, detect if systemd is running
+	d := runtime.CgroupDriver_CGROUPFS
+	if systemd.IsRunningSystemd() {
+		d = runtime.CgroupDriver_SYSTEMD
+	}
+	log.G(ctx).Debugf("no runtime handler provided cgroup driver setting, using auto-detected %s", runtime.CgroupDriver_name[int32(d)])
+	return d
+}
+
+func getCgroupDriverFromRuntimeHandlerOpts(opts interface{}) (runtime.CgroupDriver, bool) {
+	switch v := opts.(type) {
+	case *runcoptions.Options:
+		systemdCgroup := v.SystemdCgroup
+		if systemdCgroup {
+			return runtime.CgroupDriver_SYSTEMD, true
+		}
+		return runtime.CgroupDriver_CGROUPFS, true
+	}
+	return runtime.CgroupDriver_SYSTEMD, false
+}

pkg/cri/sbserver/runtime_config_linux_test.go

@@ -0,0 +1,105 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sbserver
+
+import (
+	"context"
+	"testing"
+
+	criconfig "github.com/containerd/containerd/pkg/cri/config"
+	"github.com/containerd/containerd/plugin"
+	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
+	"github.com/stretchr/testify/assert"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+func newFakeRuntimeConfig(runcV2, systemdCgroup bool) criconfig.Runtime {
+	r := criconfig.Runtime{Type: "default", Options: map[string]interface{}{}}
+	if runcV2 {
+		r.Type = plugin.RuntimeRuncV2
+		if systemdCgroup {
+			r.Options["SystemdCgroup"] = true
+		}
+	}
+	return r
+}
+
+func TestRuntimeConfig(t *testing.T) {
+	autoDetected := runtime.CgroupDriver_CGROUPFS
+	if systemd.IsRunningSystemd() {
+		autoDetected = runtime.CgroupDriver_SYSTEMD
+	}
+
+	for _, test := range []struct {
+		desc                 string
+		defaultRuntime       string
+		runtimes             map[string]criconfig.Runtime
+		expectedCgroupDriver runtime.CgroupDriver
+	}{
+		{
+			desc:                 "no runtimes",
+			expectedCgroupDriver: autoDetected,
+		},
+		{
+			desc:                 "non-runc runtime",
+			defaultRuntime:       "non-runc",
+			runtimes:             map[string]criconfig.Runtime{"non-runc": newFakeRuntimeConfig(false, false)},
+			expectedCgroupDriver: autoDetected,
+		},
+		{
+			desc: "no default, pick first in alphabetical order",
+			runtimes: map[string]criconfig.Runtime{
+				"non-runc":   newFakeRuntimeConfig(false, false),
+				"runc-2":     newFakeRuntimeConfig(true, true),
+				"runc":       newFakeRuntimeConfig(true, false),
+				"non-runc-2": newFakeRuntimeConfig(false, false),
+			},
+			expectedCgroupDriver: runtime.CgroupDriver_CGROUPFS,
+		},
+		{
+			desc:           "pick default, cgroupfs",
+			defaultRuntime: "runc-2",
+			runtimes: map[string]criconfig.Runtime{
+				"non-runc": newFakeRuntimeConfig(false, false),
+				"runc":     newFakeRuntimeConfig(true, true),
+				"runc-2":   newFakeRuntimeConfig(true, false),
+			},
+			expectedCgroupDriver: runtime.CgroupDriver_CGROUPFS,
+		},
+		{
+			desc:           "pick default, systemd",
+			defaultRuntime: "runc-2",
+			runtimes: map[string]criconfig.Runtime{
+				"non-runc": newFakeRuntimeConfig(false, false),
+				"runc":     newFakeRuntimeConfig(true, false),
+				"runc-2":   newFakeRuntimeConfig(true, true),
+			},
+			expectedCgroupDriver: runtime.CgroupDriver_SYSTEMD,
+		},
+	} {
+		test := test
+		t.Run(test.desc, func(t *testing.T) {
+			c := newTestCRIService()
+			c.config.PluginConfig.ContainerdConfig.DefaultRuntimeName = test.defaultRuntime
+			c.config.PluginConfig.ContainerdConfig.Runtimes = test.runtimes
+
+			resp, err := c.RuntimeConfig(context.TODO(), &runtime.RuntimeConfigRequest{})
+			assert.NoError(t, err)
+			assert.Equal(t, test.expectedCgroupDriver, resp.Linux.CgroupDriver, "got unexpected cgroup driver")
+		})
+	}
+}

pkg/cri/sbserver/runtime_config_other.go

@@ -0,0 +1,29 @@
+//go:build !linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sbserver
+
+import (
+	"context"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+func (c *criService) getLinuxRuntimeConfig(ctx context.Context) *runtime.LinuxRuntimeConfiguration {
+	return nil
+}

pkg/cri/server/runtime_config.go

@@ -0,0 +1,31 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+// RuntimeConfig returns configuration information of the runtime.
+func (c *criService) RuntimeConfig(ctx context.Context, r *runtime.RuntimeConfigRequest) (*runtime.RuntimeConfigResponse, error) {
+	resp := &runtime.RuntimeConfigResponse{
+		Linux: c.getLinuxRuntimeConfig(ctx),
+	}
+	return resp, nil
+}

pkg/cri/server/runtime_config_linux.go

@@ -0,0 +1,81 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"sort"
+
+	"github.com/containerd/containerd/log"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
+	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+)
+
+func (c *criService) getLinuxRuntimeConfig(ctx context.Context) *runtime.LinuxRuntimeConfiguration {
+	return &runtime.LinuxRuntimeConfiguration{CgroupDriver: c.getCgroupDriver(ctx)}
+}
+
+func (c *criService) getCgroupDriver(ctx context.Context) runtime.CgroupDriver {
+	// Go through the runtime handlers in a predictable order, starting from the
+	// default handler, others sorted in alphabetical order
+	handlerNames := make([]string, 0, len(c.config.ContainerdConfig.Runtimes))
+	for n := range c.config.ContainerdConfig.Runtimes {
+		handlerNames = append(handlerNames, n)
+	}
+	sort.Slice(handlerNames, func(i, j int) bool {
+		if handlerNames[i] == c.config.ContainerdConfig.DefaultRuntimeName {
+			return true
+		}
+		if handlerNames[j] == c.config.ContainerdConfig.DefaultRuntimeName {
+			return false
+		}
+		return handlerNames[i] < handlerNames[j]
+	})
+
+	for _, handler := range handlerNames {
+		opts, err := generateRuntimeOptions(c.config.ContainerdConfig.Runtimes[handler])
+		if err != nil {
+			log.G(ctx).Debugf("failed to parse runtime handler options for %q", handler)
+			continue
+		}
+		if d, ok := getCgroupDriverFromRuntimeHandlerOpts(opts); ok {
+			return d
+		}
+		log.G(ctx).Debugf("runtime handler %q does not provide cgroup driver information", handler)
+	}
+
+	// If no runtime handlers have a setting, detect if systemd is running
+	d := runtime.CgroupDriver_CGROUPFS
+	if systemd.IsRunningSystemd() {
+		d = runtime.CgroupDriver_SYSTEMD
+	}
+	log.G(ctx).Debugf("no runtime handler provided cgroup driver setting, using auto-detected %s", runtime.CgroupDriver_name[int32(d)])
+	return d
+}
+
+func getCgroupDriverFromRuntimeHandlerOpts(opts interface{}) (runtime.CgroupDriver, bool) {
+	switch v := opts.(type) {
+	case *runcoptions.Options:
+		systemdCgroup := v.SystemdCgroup
+		if systemdCgroup {
+			return runtime.CgroupDriver_SYSTEMD, true
+		}
+		return runtime.CgroupDriver_CGROUPFS, true
+	}
+	return runtime.CgroupDriver_SYSTEMD, false
+}