feat(platform): support dynamic registry (tkestack#2083)

api/platform/types.go

@@ -123,6 +123,8 @@ const (
 	AnywhereApplicationAnno = "tkestack.io/anywhere-application"
 	// AnywhereValidateAnno is exist, the cluster will always return validate result
 	AnywhereValidateAnno = "tkestack.io/anywhere-validate"
+	// LocationBasedImagePrefixAnno is exist, the cluster will use it as k8s images prefix
+	LocationBaseImagePrefixAnno = "tkestack.io/location-based-image-prefix"
 )
 
 // KubeVendorType describe the kubernetes provider of the cluster

api/platform/v1/types.go

@@ -134,6 +134,8 @@ const (
 	AnywhereApplicationAnno = "tkestack.io/anywhere-application"
 	// AnywhereValidateAnno is exist, the cluster will always return validate result
 	AnywhereValidateAnno = "tkestack.io/anywhere-validate"
+	// LocationBasedImagePrefixAnno is exist, the cluster will use it as k8s images prefix
+	LocationBasedImagePrefixAnno = "tkestack.io/location-based-image-prefix"
 )
 
 // KubeVendorType describe the kubernetes provider of the cluster

build/docker/tools/provider-res/Makefile

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations under the License.
 
 IMAGE = provider-res
-VERSION = v1.21.4-4
+VERSION = v1.21.4-5
 REGISTRY_PREFIX ?= tkestack
 
 COMMON_SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))

build/docker/tools/tke-installer/release.sh

@@ -23,7 +23,7 @@ set -o pipefail
 REGISTRY_PREFIX=${REGISTRY_PREFIX:-tkestack}
 BUILDER=${BUILDER:-default}
 VERSION=${VERSION:-$(git describe --dirty --always --tags | sed 's/-/./g')}
-PROVIDER_RES_VERSION=v1.21.4-4
+PROVIDER_RES_VERSION=v1.21.4-5
 K8S_VERSION=${PROVIDER_RES_VERSION%-*}
 DOCKER_VERSION=19.03.14
 CONTAINERD_VERSION=1.5.4

cmd/tke-installer/app/installer/images/images.go

@@ -117,7 +117,7 @@ var baseComponents = BaseComponents{
 	TKEPlatformController: containerregistry.Image{Name: "tke-platform-controller", Tag: Version},
 	TKERegistryAPI:        containerregistry.Image{Name: "tke-registry-api", Tag: Version},
 	TKERegistryController: containerregistry.Image{Name: "tke-registry-controller", Tag: Version},
-	ProviderRes:           containerregistry.Image{Name: "provider-res", Tag: "v1.21.4-4"},
+	ProviderRes:           containerregistry.Image{Name: "provider-res", Tag: "v1.21.4-5"},
 	TKEGateway:            containerregistry.Image{Name: "tke-gateway", Tag: Version},
 
 	NginxIngress:       containerregistry.Image{Name: "ingress-nginx-controller", Tag: "v1.1.3"},

pkg/platform/provider/baremetal/cluster/create.go

@@ -511,14 +511,23 @@ func (p *Provider) EnsureContainerRuntime(ctx context.Context, c *v1.Cluster) er
 	return p.EnsureContainerd(ctx, c)
 }
 
+func (p *Provider) getImagePrefix(c *v1.Cluster) string {
+	if anno, ok := c.Annotations[platformv1.LocationBasedImagePrefixAnno]; ok {
+		return anno
+	}
+	return containerregistryutil.GetPrefix()
+}
+
 func (p *Provider) EnsureContainerd(ctx context.Context, c *v1.Cluster) error {
 	insecureRegistries := []string{p.Config.Registry.Domain}
 	if c.Spec.TenantID != "" {
 		insecureRegistries = append(insecureRegistries, c.Spec.TenantID+"."+p.Config.Registry.Domain)
 	}
+	prefix := p.getImagePrefix(c)
 	option := &containerd.Option{
 		InsecureRegistries: insecureRegistries,
-		SandboxImage:       images.Get().Pause.FullName(),
+		SandboxImage:       path.Join(prefix, images.Get().Pause.BaseName()),
+		RegistryMirror:     prefix,
 	}
 	for _, machine := range c.Spec.Machines {
 		machineSSH, err := machine.SSH()
@@ -838,7 +847,11 @@ func (p *Provider) EnsureKubeadmInitPhaseKubeletStart(ctx context.Context, c *v1
 			phase += fmt.Sprintf(" --node-name=%s", c.Spec.Machines[0].IP)
 		}
 	}
-	return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), phase)
+	err = kubeadm.WriteInitConfig(machineSSH, p.getKubeadmInitConfig(c))
+	if err != nil {
+		return err
+	}
+	return kubeadm.Init(machineSSH, phase)
 }
 
 func (p *Provider) EnsureKubeadmInitPhaseCerts(ctx context.Context, c *v1.Cluster) error {
@@ -849,7 +862,7 @@ func (p *Provider) EnsureKubeadmInitPhaseCerts(ctx context.Context, c *v1.Cluste
 	if err != nil {
 		return err
 	}
-	return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "certs all")
+	return kubeadm.Init(machineSSH, "certs all")
 }
 
 func (p *Provider) EnsureKubeadmInitPhaseKubeConfig(ctx context.Context, c *v1.Cluster) error {
@@ -860,7 +873,7 @@ func (p *Provider) EnsureKubeadmInitPhaseKubeConfig(ctx context.Context, c *v1.C
 	if err != nil {
 		return err
 	}
-	return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "kubeconfig all")
+	return kubeadm.Init(machineSSH, "kubeconfig all")
 }
 
 func (p *Provider) EnsureKubeadmInitPhaseControlPlane(ctx context.Context, c *v1.Cluster) error {
@@ -871,7 +884,7 @@ func (p *Provider) EnsureKubeadmInitPhaseControlPlane(ctx context.Context, c *v1
 	if err != nil {
 		return err
 	}
-	return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "control-plane all")
+	return kubeadm.Init(machineSSH, "control-plane all")
 }
 
 func (p *Provider) EnsureKubeadmInitPhaseETCD(ctx context.Context, c *v1.Cluster) error {
@@ -882,7 +895,7 @@ func (p *Provider) EnsureKubeadmInitPhaseETCD(ctx context.Context, c *v1.Cluster
 	if err != nil {
 		return err
 	}
-	return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "etcd local")
+	return kubeadm.Init(machineSSH, "etcd local")
 }
 
 func (p *Provider) EnsureKubeadmInitPhaseUploadConfig(ctx context.Context, c *v1.Cluster) error {
@@ -893,15 +906,15 @@ func (p *Provider) EnsureKubeadmInitPhaseUploadConfig(ctx context.Context, c *v1
 	if err != nil {
 		return err
 	}
-	return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "upload-config all ")
+	return kubeadm.Init(machineSSH, "upload-config all ")
 }
 
 func (p *Provider) EnsureKubeadmInitPhaseUploadCerts(ctx context.Context, c *v1.Cluster) error {
 	machineSSH, err := c.Spec.Machines[0].SSH()
 	if err != nil {
 		return err
 	}
-	return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "upload-certs --upload-certs")
+	return kubeadm.Init(machineSSH, "upload-certs --upload-certs")
 }
 
 func (p *Provider) EnsureKubeadmInitPhaseBootstrapToken(ctx context.Context, c *v1.Cluster) error {
@@ -912,7 +925,7 @@ func (p *Provider) EnsureKubeadmInitPhaseBootstrapToken(ctx context.Context, c *
 	if err != nil {
 		return err
 	}
-	return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "bootstrap-token")
+	return kubeadm.Init(machineSSH, "bootstrap-token")
 }
 
 func (p *Provider) EnsureKubeadmInitPhaseAddon(ctx context.Context, c *v1.Cluster) error {
@@ -923,7 +936,7 @@ func (p *Provider) EnsureKubeadmInitPhaseAddon(ctx context.Context, c *v1.Cluste
 	if err != nil {
 		return err
 	}
-	return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "addon all")
+	return kubeadm.Init(machineSSH, "addon all")
 }
 
 func (p *Provider) EnsureGalaxy(ctx context.Context, c *v1.Cluster) error {

pkg/platform/provider/baremetal/cluster/kubeadm.go

@@ -21,6 +21,7 @@ package cluster
 import (
 	"fmt"
 	"net"
+	"path"
 
 	"github.com/imdario/mergo"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,7 +35,6 @@ import (
 	"tkestack.io/tke/pkg/platform/provider/baremetal/phases/kubeadm"
 	v1 "tkestack.io/tke/pkg/platform/types/v1"
 	"tkestack.io/tke/pkg/util/apiclient"
-	"tkestack.io/tke/pkg/util/containerregistry"
 	"tkestack.io/tke/pkg/util/json"
 	"tkestack.io/tke/pkg/util/version"
 )
@@ -184,7 +184,7 @@ func (p *Provider) getClusterConfiguration(c *v1.Cluster) *kubeadmv1beta2.Cluste
 		DNS: kubeadmv1beta2.DNS{
 			Type: kubeadmv1beta2.CoreDNS,
 		},
-		ImageRepository: containerregistry.GetPrefix(),
+		ImageRepository: p.getImagePrefix(c),
 		ClusterName:     c.Name,
 		FeatureGates: map[string]bool{
 			"IPv6DualStack": c.Cluster.Spec.Features.IPv6DualStack},
@@ -311,7 +311,7 @@ func (p *Provider) getSchedulerExtraArgs(c *v1.Cluster) map[string]string {
 
 func (p *Provider) getKubeletExtraArgs(c *v1.Cluster) map[string]string {
 	args := map[string]string{
-		"pod-infra-container-image": images.Get().Pause.FullName(),
+		"pod-infra-container-image": path.Join(p.getImagePrefix(c), images.Get().Pause.BaseName()),
 	}
 
 	utilruntime.Must(mergo.Merge(&args, c.Spec.KubeletExtraArgs))

pkg/platform/provider/baremetal/cluster/update.go

@@ -119,7 +119,12 @@ func (p *Provider) EnsureAPIServerCert(ctx context.Context, c *v1.Cluster) error
 			preActions = append(preActions, fmt.Sprintf("rm -f %s", file))
 		}
 
-		err = kubeadm.Init(s, kubeadmConfig, "certs apiserver", preActions...)
+		err = kubeadm.WriteInitConfig(s, kubeadmConfig)
+		if err != nil {
+			return errors.Wrap(err, machine.IP)
+		}
+
+		err = kubeadm.Init(s, "certs apiserver", preActions...)
 		if err != nil {
 			return errors.Wrap(err, machine.IP)
 		}

pkg/platform/provider/baremetal/conf/containerd/config.toml

@@ -42,5 +42,5 @@ state = "/run/containerd"
           insecure_skip_verify=true
         {{end}}
       [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."ccr.ccs.tencentyun.com"]
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{.RegistryMirror}}"]
           endpoint = ["http://mirrors.tke.com"]

pkg/platform/provider/baremetal/config/config.go

@@ -57,6 +57,7 @@ type Config struct {
 	Scheduler                  Scheduler         `yaml:"scheduler"`
 	AuthzWebhook               AuthzWebhook      `yaml:"authzWebhook"`
 	Business                   Business          `yaml:"business"`
+	SupportOSList              []string          `yaml:"supportOSList"`
 }
 
 func (c *Config) Save(filename string) error {

pkg/platform/provider/baremetal/machine/create.go

@@ -49,6 +49,7 @@ import (
 	typesv1 "tkestack.io/tke/pkg/platform/types/v1"
 	"tkestack.io/tke/pkg/util/apiclient"
 	"tkestack.io/tke/pkg/util/cmdstring"
+	containerregistryutil "tkestack.io/tke/pkg/util/containerregistry"
 	"tkestack.io/tke/pkg/util/hosts"
 )
 
@@ -338,6 +339,7 @@ func (p *Provider) EnsureContainerd(ctx context.Context, machine *platformv1.Mac
 		InsecureRegistries: insecureRegistries,
 		IsGPU:              gpu.IsEnable(machine.Spec.Labels),
 		SandboxImage:       images.Get().Pause.FullName(),
+		RegistryMirror:     containerregistryutil.GetPrefix(),
 	}
 	err = containerd.Install(machineSSH, option)
 	if err != nil {

pkg/platform/provider/baremetal/phases/containerd/containerd.go

@@ -37,6 +37,7 @@ type Option struct {
 	IsGPU              bool
 	Root               string
 	SandboxImage       string
+	RegistryMirror     string
 }
 
 const (

pkg/platform/provider/baremetal/phases/kubeadm/kubeadm.go

@@ -102,16 +102,15 @@ func Install(s ssh.Interface, option *Option) error {
 	return nil
 }
 
-func Init(s ssh.Interface, kubeadmConfig *InitConfig, phase string, preActions ...string) error {
+func WriteInitConfig(s ssh.Interface, kubeadmConfig *InitConfig) error {
 	configData, err := kubeadmConfig.Marshal()
 	if err != nil {
 		return err
 	}
-	err = s.WriteFile(bytes.NewReader(configData), constants.KubeadmConfigFileName)
-	if err != nil {
-		return err
-	}
+	return s.WriteFile(bytes.NewReader(configData), constants.KubeadmConfigFileName)
+}
 
+func Init(s ssh.Interface, phase string, preActions ...string) error {
 	cmd, err := template.ParseString(initCmd, map[string]interface{}{
 		"Phase":  phase,
 		"Config": constants.KubeadmConfigFileName,

pkg/platform/provider/baremetal/validation/cluster.go

@@ -542,6 +542,7 @@ func ValidateCIDRs(cls *platform.Cluster, specPath *field.Path) field.ErrorList
 		result.Name = AnywhereValidateItemHostNetOverlapping
 		result.Description = "Verify Node IP(s) and CIDR Config"
 		result.ErrorList = allErrs
+		result.Checked = true
 
 		return field.ErrorList{result.ToFieldError()}