Merge pull request kubernetes#56280 from xiangpengzhao/kubeadm-valida…

…te-kc Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Add validation of kubelet and kube-proxy configuration in kubeadm. **What this PR does / why we need it**: kubeadm has implemented the support for Kubelet Dynamic Configuration. This PR adds validation on the kubelet configuration. kube-proxy validation also added in this PR. **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: ref: kubernetes/kubeadm#28 **Special notes for your reviewer**: /cc @luxas @kubernetes/sig-cluster-lifecycle-pr-reviews **Release note**: ```release-note NONE ```
madhanrm · Nov 24, 2017 · 45db5e7 · 45db5e7
2 parents 64ccd76 + cf97d6f
commit 45db5e7
Show file tree

Hide file tree

Showing 5 changed files with 178 additions and 34 deletions.
diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD
@@ -1,9 +1,4 @@
-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-)
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
@@ -23,9 +18,12 @@ go_library(
         "//conditions:default": [],
     }),
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1",
+    visibility = ["//visibility:public"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/features:go_default_library",
+        "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
         "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
         "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library",
         "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
@@ -48,4 +46,5 @@ filegroup(
     name = "all-srcs",
     srcs = [":package-srcs"],
     tags = ["automanaged"],
+    visibility = ["//visibility:public"],
 )
diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go
@@ -23,6 +23,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
+	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
 	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
 	kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
 	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
@@ -110,7 +112,9 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
 	}
 
 	SetDefaultsEtcdSelfHosted(obj)
-	SetDefaults_KubeletConfiguration(obj)
+	if features.Enabled(obj.FeatureGates, features.DynamicKubeletConfig) {
+		SetDefaults_KubeletConfiguration(obj)
+	}
 	SetDefaults_ProxyConfiguration(obj)
 }
 
@@ -200,4 +204,9 @@ func SetDefaults_KubeletConfiguration(obj *MasterConfiguration) {
 	if obj.KubeletConfiguration.BaseConfig.CAdvisorPort == nil {
 		obj.KubeletConfiguration.BaseConfig.CAdvisorPort = utilpointer.Int32Ptr(0)
 	}
+
+	scheme, _, _ := kubeletscheme.NewSchemeAndCodecs()
+	if scheme != nil {
+		scheme.Default(obj.KubeletConfiguration.BaseConfig)
+	}
 }
diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/BUILD b/cmd/kubeadm/app/apis/kubeadm/validation/BUILD
@@ -1,30 +1,10 @@
-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-    "go_test",
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = ["validation_test.go"],
-    importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation",
-    library = ":go_default_library",
-    deps = [
-        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
-        "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
-        "//pkg/util/pointer:go_default_library",
-        "//vendor/github.com/spf13/pflag:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
-    ],
-)
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
     srcs = ["validation.go"],
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation",
+    visibility = ["//visibility:public"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
@@ -33,6 +13,9 @@ go_library(
         "//cmd/kubeadm/app/util/token:go_default_library",
         "//pkg/apis/core/validation:go_default_library",
         "//pkg/kubeapiserver/authorizer/modes:go_default_library",
+        "//pkg/kubelet/apis/kubeletconfig:go_default_library",
+        "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
+        "//pkg/kubelet/apis/kubeletconfig/validation:go_default_library",
         "//pkg/proxy/apis/kubeproxyconfig:go_default_library",
         "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library",
         "//pkg/proxy/apis/kubeproxyconfig/validation:go_default_library",
@@ -45,6 +28,22 @@ go_library(
     ],
 )
 
+go_test(
+    name = "go_default_test",
+    srcs = ["validation_test.go"],
+    importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation",
+    library = ":go_default_library",
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
+        "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
+        "//pkg/util/pointer:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+    ],
+)
+
 filegroup(
     name = "package-srcs",
     srcs = glob(["**"]),
@@ -56,4 +55,5 @@ filegroup(
     name = "all-srcs",
     srcs = [":package-srcs"],
     tags = ["automanaged"],
+    visibility = ["//visibility:public"],
 )
diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go
@@ -36,6 +36,9 @@ import (
 	tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
 	apivalidation "k8s.io/kubernetes/pkg/apis/core/validation"
 	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
+	kubeletvalidation "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/validation"
 	"k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
 	kubeproxyscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
 	proxyvalidation "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/validation"
@@ -74,7 +77,10 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList
 	allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...)
 	allErrs = append(allErrs, ValidateFeatureGates(c.FeatureGates, field.NewPath("feature-gates"))...)
 	allErrs = append(allErrs, ValidateAPIEndpoint(c, field.NewPath("api-endpoint"))...)
-	//allErrs = append(allErrs, ValidateProxy(c, field.NewPath("kube-proxy"))...)
+	allErrs = append(allErrs, ValidateProxy(c, field.NewPath("kube-proxy"))...)
+	if features.Enabled(c.FeatureGates, features.DynamicKubeletConfig) {
+		allErrs = append(allErrs, ValidateKubeletConfiguration(&c.KubeletConfiguration, field.NewPath("kubeletConfiguration"))...)
+	}
 	return allErrs
 }
 
@@ -369,3 +375,29 @@ func ValidateIgnorePreflightErrors(ignorePreflightErrors []string, skipPreflight
 
 	return ignoreErrors, allErrs.ToAggregate()
 }
+
+// ValidateKubeletConfiguration validates kubelet configuration and collects all encountered errors
+func ValidateKubeletConfiguration(c *kubeadm.KubeletConfiguration, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	scheme, _, err := kubeletscheme.NewSchemeAndCodecs()
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fldPath, "kubeletConfiguration", err.Error()))
+		return allErrs
+	}
+
+	// Convert versioned config to internal config
+	internalcfg := &kubeletconfig.KubeletConfiguration{}
+	err = scheme.Convert(c.BaseConfig, internalcfg, nil)
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fldPath, "kubeletConfiguration", err.Error()))
+		return allErrs
+	}
+
+	err = kubeletvalidation.ValidateKubeletConfiguration(internalcfg)
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fldPath, "kubeletConfiguration", err.Error()))
+	}
+
+	return allErrs
+}
diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go
@@ -25,8 +25,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
 	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
-	"k8s.io/kubernetes/pkg/util/pointer"
+	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
 )
 
 func TestValidateTokenDiscovery(t *testing.T) {
@@ -337,10 +338,25 @@ func TestValidateMasterConfiguration(t *testing.T) {
 				},
 				KubeProxy: kubeadm.KubeProxy{
 					Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{
+						BindAddress:        "192.168.59.103",
+						HealthzBindAddress: "0.0.0.0:10256",
+						MetricsBindAddress: "127.0.0.1:10249",
+						ClusterCIDR:        "192.168.59.0/24",
+						UDPIdleTimeout:     metav1.Duration{Duration: 1 * time.Second},
+						ConfigSyncPeriod:   metav1.Duration{Duration: 1 * time.Second},
+						IPTables: kubeproxyconfigv1alpha1.KubeProxyIPTablesConfiguration{
+							MasqueradeAll: true,
+							SyncPeriod:    metav1.Duration{Duration: 5 * time.Second},
+							MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second},
+						},
+						IPVS: kubeproxyconfigv1alpha1.KubeProxyIPVSConfiguration{
+							SyncPeriod:    metav1.Duration{Duration: 10 * time.Second},
+							MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second},
+						},
 						Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{
-							Max:        pointer.Int32Ptr(2),
-							MaxPerCore: pointer.Int32Ptr(1),
-							Min:        pointer.Int32Ptr(1),
+							Max:        utilpointer.Int32Ptr(2),
+							MaxPerCore: utilpointer.Int32Ptr(1),
+							Min:        utilpointer.Int32Ptr(1),
 							TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second},
 							TCPCloseWaitTimeout:   &metav1.Duration{Duration: 5 * time.Second},
 						},
@@ -361,6 +377,32 @@ func TestValidateMasterConfiguration(t *testing.T) {
 					AdvertiseAddress: "1:2:3::4",
 					BindPort:         3446,
 				},
+				KubeProxy: kubeadm.KubeProxy{
+					Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{
+						BindAddress:        "192.168.59.103",
+						HealthzBindAddress: "0.0.0.0:10256",
+						MetricsBindAddress: "127.0.0.1:10249",
+						ClusterCIDR:        "192.168.59.0/24",
+						UDPIdleTimeout:     metav1.Duration{Duration: 1 * time.Second},
+						ConfigSyncPeriod:   metav1.Duration{Duration: 1 * time.Second},
+						IPTables: kubeproxyconfigv1alpha1.KubeProxyIPTablesConfiguration{
+							MasqueradeAll: true,
+							SyncPeriod:    metav1.Duration{Duration: 5 * time.Second},
+							MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second},
+						},
+						IPVS: kubeproxyconfigv1alpha1.KubeProxyIPVSConfiguration{
+							SyncPeriod:    metav1.Duration{Duration: 10 * time.Second},
+							MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second},
+						},
+						Conntrack: kubeproxyconfigv1alpha1.KubeProxyConntrackConfiguration{
+							Max:        utilpointer.Int32Ptr(2),
+							MaxPerCore: utilpointer.Int32Ptr(1),
+							Min:        utilpointer.Int32Ptr(1),
+							TCPEstablishedTimeout: &metav1.Duration{Duration: 5 * time.Second},
+							TCPCloseWaitTimeout:   &metav1.Duration{Duration: 5 * time.Second},
+						},
+					},
+				},
 				AuthorizationModes: []string{"Node", "RBAC"},
 				Networking: kubeadm.Networking{
 					ServiceSubnet: "2001:db8::1/98",
@@ -502,3 +544,65 @@ func TestValidateIgnorePreflightErrors(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateKubeletConfiguration(t *testing.T) {
+	successCase := &kubeadm.KubeletConfiguration{
+		BaseConfig: &kubeletconfigv1alpha1.KubeletConfiguration{
+			CgroupsPerQOS:               utilpointer.BoolPtr(true),
+			EnforceNodeAllocatable:      []string{"pods", "system-reserved", "kube-reserved"},
+			SystemCgroups:               "",
+			CgroupRoot:                  "",
+			CAdvisorPort:                utilpointer.Int32Ptr(0),
+			EventBurst:                  10,
+			EventRecordQPS:              utilpointer.Int32Ptr(5),
+			HealthzPort:                 utilpointer.Int32Ptr(10248),
+			ImageGCHighThresholdPercent: utilpointer.Int32Ptr(85),
+			ImageGCLowThresholdPercent:  utilpointer.Int32Ptr(80),
+			IPTablesDropBit:             utilpointer.Int32Ptr(15),
+			IPTablesMasqueradeBit:       utilpointer.Int32Ptr(14),
+			KubeAPIBurst:                10,
+			KubeAPIQPS:                  utilpointer.Int32Ptr(5),
+			MaxOpenFiles:                1000000,
+			MaxPods:                     110,
+			OOMScoreAdj:                 utilpointer.Int32Ptr(-999),
+			PodsPerCore:                 100,
+			Port:                        65535,
+			ReadOnlyPort:                utilpointer.Int32Ptr(0),
+			RegistryBurst:               10,
+			RegistryPullQPS:             utilpointer.Int32Ptr(5),
+		},
+	}
+	if allErrors := ValidateKubeletConfiguration(successCase, nil); len(allErrors) != 0 {
+		t.Errorf("failed ValidateKubeletConfiguration: expect no errors but got %v", allErrors)
+	}
+
+	errorCase := &kubeadm.KubeletConfiguration{
+		BaseConfig: &kubeletconfigv1alpha1.KubeletConfiguration{
+			CgroupsPerQOS:               utilpointer.BoolPtr(false),
+			EnforceNodeAllocatable:      []string{"pods", "system-reserved", "kube-reserved", "illegal-key"},
+			SystemCgroups:               "/",
+			CgroupRoot:                  "",
+			CAdvisorPort:                utilpointer.Int32Ptr(-10),
+			EventBurst:                  -10,
+			EventRecordQPS:              utilpointer.Int32Ptr(-10),
+			HealthzPort:                 utilpointer.Int32Ptr(-10),
+			ImageGCHighThresholdPercent: utilpointer.Int32Ptr(101),
+			ImageGCLowThresholdPercent:  utilpointer.Int32Ptr(101),
+			IPTablesDropBit:             utilpointer.Int32Ptr(-10),
+			IPTablesMasqueradeBit:       utilpointer.Int32Ptr(-10),
+			KubeAPIBurst:                -10,
+			KubeAPIQPS:                  utilpointer.Int32Ptr(-10),
+			MaxOpenFiles:                -10,
+			MaxPods:                     -10,
+			OOMScoreAdj:                 utilpointer.Int32Ptr(-1001),
+			PodsPerCore:                 -10,
+			Port:                        0,
+			ReadOnlyPort:                utilpointer.Int32Ptr(-10),
+			RegistryBurst:               -10,
+			RegistryPullQPS:             utilpointer.Int32Ptr(-10),
+		},
+	}
+	if allErrors := ValidateKubeletConfiguration(errorCase, nil); len(allErrors) == 0 {
+		t.Errorf("failed ValidateKubeletConfiguration: expect errors but got no error")
+	}
+}