diff --git a/pkg/controller/bootstrap/bootstrap.go b/pkg/controller/bootstrap/bootstrap.go index c18f391a8e..5f0a0a5b1f 100644 --- a/pkg/controller/bootstrap/bootstrap.go +++ b/pkg/controller/bootstrap/bootstrap.go @@ -20,7 +20,9 @@ import ( apicfgv1 "github.com/openshift/api/config/v1" apioperatorsv1alpha1 "github.com/openshift/api/operator/v1alpha1" mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" containerruntimeconfig "github.com/openshift/machine-config-operator/pkg/controller/container-runtime-config" + kubeletconfig "github.com/openshift/machine-config-operator/pkg/controller/kubelet-config" "github.com/openshift/machine-config-operator/pkg/controller/render" "github.com/openshift/machine-config-operator/pkg/controller/template" ) @@ -46,6 +48,7 @@ func New(templatesDir, manifestDir, pullSecretFile string) *Bootstrap { // Run runs boostrap for Machine Config Controller // It writes all the assets to destDir +// nolint:gocyclo func (b *Bootstrap) Run(destDir string) error { infos, err := ioutil.ReadDir(b.manifestDir) if err != nil { @@ -70,6 +73,7 @@ func (b *Bootstrap) Run(destDir string) error { decoder := codecFactory.UniversalDecoder(mcfgv1.GroupVersion, apioperatorsv1alpha1.GroupVersion, apicfgv1.GroupVersion) var cconfig *mcfgv1.ControllerConfig + var featureGate *apicfgv1.FeatureGate var pools []*mcfgv1.MachineConfigPool var configs []*mcfgv1.MachineConfig var icspRules []*apioperatorsv1alpha1.ImageContentSourcePolicy @@ -112,6 +116,10 @@ func (b *Bootstrap) Run(destDir string) error { icspRules = append(icspRules, obj) case *apicfgv1.Image: imgCfg = obj + case *apicfgv1.FeatureGate: + if obj.GetName() == ctrlcommon.ClusterFeatureInstanceName { + featureGate = obj + } default: glog.Infof("skipping %q [%d] manifest because of unhandled %T", file.Name(), idx+1, obji) } @@ -121,7 +129,7 @@ func (b *Bootstrap) Run(destDir string) error { if cconfig == nil { return fmt.Errorf("error: no controllerconfig found in dir: %q", destDir) } - iconfigs, err := template.RunBootstrap(b.templatesDir, cconfig, psraw) + iconfigs, err := template.RunBootstrap(b.templatesDir, cconfig, psraw, featureGate) if err != nil { return err } @@ -133,6 +141,14 @@ func (b *Bootstrap) Run(destDir string) error { } configs = append(configs, rconfigs...) + if featureGate != nil { + kConfigs, err := kubeletconfig.RunFeatureGateBootstrap(b.templatesDir, featureGate, cconfig, pools) + if err != nil { + return err + } + configs = append(configs, kConfigs...) + } + fpools, gconfigs, err := render.RunBootstrap(pools, configs, cconfig) if err != nil { return err diff --git a/pkg/controller/bootstrap/bootstrap_test.go b/pkg/controller/bootstrap/bootstrap_test.go index 4d5fe08743..469a0d070b 100644 --- a/pkg/controller/bootstrap/bootstrap_test.go +++ b/pkg/controller/bootstrap/bootstrap_test.go @@ -44,6 +44,19 @@ spec: want: []manifest{{ Raw: []byte(`{"apiVersion":"extensions/v1beta1","kind":"Ingress","metadata":{"name":"test-ingress","namespace":"test-namespace"},"spec":{"rules":[{"http":{"paths":[{"backend":{"serviceName":"test","servicePort":80},"path":"/testpath"}]}}]}}`), }}, + }, { + name: "feature gate", + raw: ` +apiVersion: config.openshift.io/v1 +kind: FeatureGate +metadata: + name: cluster +spec: + featureSet: TechPreviewNoUpgrade +`, + want: []manifest{{ + Raw: []byte(`{"apiVersion":"config.openshift.io/v1","kind":"FeatureGate","metadata":{"name":"cluster"},"spec":{"featureSet":"TechPreviewNoUpgrade"}}`), + }}, }, { name: "two-resources", raw: ` diff --git a/pkg/controller/kubelet-config/helpers.go b/pkg/controller/kubelet-config/helpers.go index cddfb91765..7a1a10af53 100644 --- a/pkg/controller/kubelet-config/helpers.go +++ b/pkg/controller/kubelet-config/helpers.go @@ -311,3 +311,13 @@ func newKubeletconfigJSONEncoder(targetVersion schema.GroupVersion) (runtime.Enc } return codecs.EncoderForVersion(info.Serializer, targetVersion), nil } + +// kubeletConfigToIgnFile converts a KubeletConfiguration to an Ignition File +func kubeletConfigToIgnFile(cfg *kubeletconfigv1beta1.KubeletConfiguration) (*ign3types.File, error) { + cfgJSON, err := EncodeKubeletConfig(cfg, kubeletconfigv1beta1.SchemeGroupVersion) + if err != nil { + return nil, fmt.Errorf("could not encode kubelet configuration: %v", err) + } + cfgIgn := createNewKubeletIgnition(cfgJSON) + return cfgIgn, nil +} diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index 814f09d1b0..e69a18cecc 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -330,14 +330,43 @@ func (ctrl *Controller) handleFeatureErr(err error, key interface{}) { ctrl.featureQueue.AddAfter(key, 1*time.Minute) } -func (ctrl *Controller) generateOriginalKubeletConfig(role string, featureGate *configv1.FeatureGate) (*ign3types.File, error) { - cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) +// generateOriginalKubeletConfigWithFeatureGates generates a KubeletConfig and ensure the correct feature gates are set +// based on the given FeatureGate. +func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, templatesDir, role string, features *configv1.FeatureGate) (*kubeletconfigv1beta1.KubeletConfiguration, error) { + originalKubeletIgn, err := generateOriginalKubeletConfigIgn(cc, templatesDir, role, features) if err != nil { - return nil, fmt.Errorf("could not get ControllerConfig %v", err) + return nil, fmt.Errorf("could not generate the original Kubelet config ignition: %v", err) } + if originalKubeletIgn.Contents.Source == nil { + return nil, fmt.Errorf("the original Kubelet source string is empty: %v", err) + } + dataURL, err := dataurl.DecodeString(*originalKubeletIgn.Contents.Source) + if err != nil { + return nil, fmt.Errorf("could not decode the original Kubelet source string: %v", err) + } + originalKubeConfig, err := decodeKubeletConfig(dataURL.Data) + if err != nil { + return nil, fmt.Errorf("could not deserialize the Kubelet source: %v", err) + } + + featureGates, err := generateFeatureMap(features, openshiftOnlyFeatureGates...) + if err != nil { + return nil, fmt.Errorf("could not generate features map: %v", err) + } + + // Merge in Feature Gates. + // If they are the same, this will be a no-op + if err := mergo.Merge(&originalKubeConfig.FeatureGates, featureGates, mergo.WithOverride); err != nil { + return nil, fmt.Errorf("could not merge feature gates: %v", err) + } + + return originalKubeConfig, nil +} + +func generateOriginalKubeletConfigIgn(cc *mcfgv1.ControllerConfig, templatesDir, role string, featureGate *configv1.FeatureGate) (*ign3types.File, error) { // Render the default templates rc := &mtmpl.RenderConfig{ControllerConfigSpec: &cc.Spec, FeatureGate: featureGate} - generatedConfigs, err := mtmpl.GenerateMachineConfigsForRole(rc, role, ctrl.templatesDir) + generatedConfigs, err := mtmpl.GenerateMachineConfigsForRole(rc, role, templatesDir) if err != nil { return nil, fmt.Errorf("GenerateMachineConfigsforRole failed with error %s", err) } @@ -478,12 +507,6 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { err := fmt.Errorf("could not fetch FeatureGates: %v", err) return ctrl.syncStatusOnly(cfg, err) } - featureGates, err := generateFeatureMap(features) - if err != nil { - err := fmt.Errorf("could not generate FeatureMap: %v", err) - glog.V(2).Infof("%v", err) - return ctrl.syncStatusOnly(cfg, err) - } for _, pool := range mcpPools { if pool.Spec.Configuration.Name == "" { @@ -512,20 +535,14 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { userDefinedSystemReserved := make(map[string]string, 2) // Generate the original KubeletConfig - originalKubeletIgn, err := ctrl.generateOriginalKubeletConfig(role, features) - if err != nil { - return ctrl.syncStatusOnly(cfg, err, "could not generate the original Kubelet config: %v", err) - } - if originalKubeletIgn.Contents.Source == nil { - return ctrl.syncStatusOnly(cfg, err, "the original Kubelet source string is empty: %v", err) - } - dataURL, err := dataurl.DecodeString(*originalKubeletIgn.Contents.Source) + cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) if err != nil { - return ctrl.syncStatusOnly(cfg, err, "could not decode the original Kubelet source string: %v", err) + return fmt.Errorf("could not get ControllerConfig %v", err) } - originalKubeConfig, err := decodeKubeletConfig(dataURL.Data) + + originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, features) if err != nil { - return ctrl.syncStatusOnly(cfg, err, "could not deserialize the Kubelet source: %v", err) + return ctrl.syncStatusOnly(cfg, err, "could not get original kubelet config: %v", err) } // Get the default API Server Security Profile @@ -561,29 +578,21 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { delete(specKubeletConfig.SystemReserved, "cpu") } + // FeatureGates must be set from the FeatureGate. + // Remove them here to prevent the specKubeletConfig merge overwriting them. + specKubeletConfig.FeatureGates = nil + // Merge the Old and New err = mergo.Merge(originalKubeConfig, specKubeletConfig, mergo.WithOverride) if err != nil { return ctrl.syncStatusOnly(cfg, err, "could not merge original config and new config: %v", err) } - // Merge in Feature Gates - err = mergo.Merge(&originalKubeConfig.FeatureGates, featureGates, mergo.WithOverride) - if err != nil { - return ctrl.syncStatusOnly(cfg, err, "could not merge FeatureGates: %v", err) - } - // Encode the new config into raw JSON - cfgJSON, err := EncodeKubeletConfig(originalKubeConfig, kubeletconfigv1beta1.SchemeGroupVersion) - if err != nil { - return ctrl.syncStatusOnly(cfg, err, "could not encode JSON: %v", err) - } - kubeletIgnition = createNewKubeletIgnition(cfgJSON) - } else { - // Encode the new config into raw JSON - cfgJSON, err := EncodeKubeletConfig(originalKubeConfig, kubeletconfigv1beta1.SchemeGroupVersion) - if err != nil { - return ctrl.syncStatusOnly(cfg, err, "could not encode JSON: %v", err) - } - kubeletIgnition = createNewKubeletIgnition(cfgJSON) + } + + // Encode the new config into an Ignition File + kubeletIgnition, err = kubeletConfigToIgnFile(originalKubeConfig) + if err != nil { + return ctrl.syncStatusOnly(cfg, err, "could not encode JSON: %v", err) } if isNotFound { diff --git a/pkg/controller/kubelet-config/kubelet_config_features.go b/pkg/controller/kubelet-config/kubelet_config_features.go index 965a79f8e5..9375328251 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features.go +++ b/pkg/controller/kubelet-config/kubelet_config_features.go @@ -8,18 +8,16 @@ import ( "github.com/clarketm/json" "github.com/golang/glog" - "github.com/imdario/mergo" osev1 "github.com/openshift/api/config/v1" "github.com/openshift/library-go/pkg/cloudprovider" - "github.com/vincent-petithory/dataurl" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/retry" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" "github.com/openshift/machine-config-operator/pkg/version" ) @@ -69,9 +67,10 @@ func (ctrl *Controller) syncFeatureHandler(key string) error { } else if err != nil { return err } - featureGates, err := generateFeatureMap(features, openshiftOnlyFeatureGates...) + + cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) if err != nil { - return err + return fmt.Errorf("could not get ControllerConfig %v", err) } // Find all MachineConfigPools @@ -100,43 +99,15 @@ func (ctrl *Controller) syncFeatureHandler(key string) error { return err } } - // Generate the original KubeletConfig - originalKubeletIgn, err := ctrl.generateOriginalKubeletConfig(role, nil) - if err != nil { - return err - } - if originalKubeletIgn.Contents.Source == nil { - return fmt.Errorf("could not find original Kubelet config to decode") - } - dataURL, err := dataurl.DecodeString(*originalKubeletIgn.Contents.Source) - if err != nil { - return err - } - originalKubeConfig, err := decodeKubeletConfig(dataURL.Data) + + rawCfgIgn, err := generateKubeConfigIgnFromFeatures(cc, ctrl.templatesDir, role, features) if err != nil { return err } - // Check to see if FeatureGates are equal - if reflect.DeepEqual(originalKubeConfig.FeatureGates, *featureGates) { + if rawCfgIgn == nil { continue } - // Merge in Feature Gates - err = mergo.Merge(&originalKubeConfig.FeatureGates, featureGates, mergo.WithOverride) - if err != nil { - return err - } - // Encode the new config into raw JSON - cfgJSON, err := EncodeKubeletConfig(originalKubeConfig, kubeletconfigv1beta1.SchemeGroupVersion) - if err != nil { - return err - } - tempIgnConfig := ctrlcommon.NewIgnConfig() - cfgIgn := createNewKubeletIgnition(cfgJSON) - tempIgnConfig.Storage.Files = append(tempIgnConfig.Storage.Files, *cfgIgn) - rawCfgIgn, err := json.Marshal(tempIgnConfig) - if err != nil { - return err - } + mc.Spec.Config.Raw = rawCfgIgn mc.ObjectMeta.Annotations = map[string]string{ ctrlcommon.GeneratedByControllerVersionAnnotationKey: version.Hash, @@ -231,3 +202,70 @@ func generateFeatureMap(features *osev1.FeatureGate, exclusions ...string) (*map } return &rv, nil } + +func generateKubeConfigIgnFromFeatures(cc *mcfgv1.ControllerConfig, templatesDir, role string, features *osev1.FeatureGate) ([]byte, error) { + originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, templatesDir, role, features) + if err != nil { + return nil, err + } + defaultFeatures, err := generateFeatureMap(createNewDefaultFeatureGate(), openshiftOnlyFeatureGates...) + if err != nil { + return nil, err + } + + // Check to see if configured FeatureGates are equivalent to the Default FeatureSet. + if reflect.DeepEqual(originalKubeConfig.FeatureGates, *defaultFeatures) { + // When there is no difference, this isn't an error, but no machine config should be created + return nil, nil + } + + // Encode the new config into raw JSON + cfgIgn, err := kubeletConfigToIgnFile(originalKubeConfig) + if err != nil { + return nil, err + } + + tempIgnConfig := ctrlcommon.NewIgnConfig() + tempIgnConfig.Storage.Files = append(tempIgnConfig.Storage.Files, *cfgIgn) + rawCfgIgn, err := json.Marshal(tempIgnConfig) + if err != nil { + return nil, err + } + return rawCfgIgn, nil +} + +func RunFeatureGateBootstrap(templateDir string, features *osev1.FeatureGate, controllerConfig *mcfgv1.ControllerConfig, mcpPools []*mcfgv1.MachineConfigPool) ([]*mcfgv1.MachineConfig, error) { + machineConfigs := []*mcfgv1.MachineConfig{} + + for _, pool := range mcpPools { + role := pool.Name + rawCfgIgn, err := generateKubeConfigIgnFromFeatures(controllerConfig, templateDir, role, features) + if err != nil { + return nil, err + } + if rawCfgIgn == nil { + continue + } + + // Get MachineConfig + managedKey, err := getManagedFeaturesKey(pool, nil) + if err != nil { + return nil, err + } + + ignConfig := ctrlcommon.NewIgnConfig() + mc, err := ctrlcommon.MachineConfigFromIgnConfig(role, managedKey, ignConfig) + if err != nil { + return nil, err + } + + mc.Spec.Config.Raw = rawCfgIgn + mc.ObjectMeta.Annotations = map[string]string{ + ctrlcommon.GeneratedByControllerVersionAnnotationKey: version.Hash, + } + + machineConfigs = append(machineConfigs, mc) + } + + return machineConfigs, nil +} diff --git a/pkg/controller/kubelet-config/kubelet_config_features_test.go b/pkg/controller/kubelet-config/kubelet_config_features_test.go index 6625468d13..ecc7cbbf17 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features_test.go +++ b/pkg/controller/kubelet-config/kubelet_config_features_test.go @@ -11,8 +11,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" "github.com/openshift/machine-config-operator/test/helpers" + "github.com/stretchr/testify/require" ) func TestFeatureGateDrift(t *testing.T) { @@ -23,7 +25,7 @@ func TestFeatureGateDrift(t *testing.T) { f.ccLister = append(f.ccLister, cc) ctrl := f.newController() - kubeletConfig, err := ctrl.generateOriginalKubeletConfig("master", nil) + kubeletConfig, err := generateOriginalKubeletConfigIgn(cc, ctrl.templatesDir, "master", nil) if err != nil { t.Errorf("could not generate kubelet config from templates %v", err) } @@ -131,3 +133,78 @@ func TestFeaturesCustomNoUpgrade(t *testing.T) { }) } } + +func TestBootstrapFeaturesDefault(t *testing.T) { + for _, platform := range []configv1.PlatformType{configv1.AWSPlatformType, configv1.NonePlatformType, "unrecognized"} { + t.Run(string(platform), func(t *testing.T) { + + cc := newControllerConfig(ctrlcommon.ControllerConfigName, platform) + mcp := helpers.NewMachineConfigPool("master", nil, helpers.MasterSelector, "v0") + mcp2 := helpers.NewMachineConfigPool("worker", nil, helpers.WorkerSelector, "v0") + mcps := []*mcfgv1.MachineConfigPool{mcp, mcp2} + + features := createNewDefaultFeatureGate() + + mcs, err := RunFeatureGateBootstrap("../../../templates", features, cc, mcps) + if err != nil { + t.Errorf("could not run feature gate bootstrap: %v", err) + } + if len(mcs) > 0 { + t.Errorf("expected no machine config generated with the default feature gate, got %d configs", len(mcs)) + } + }) + } +} + +func TestBootstrapFeaturesCustomNoUpgrade(t *testing.T) { + for _, platform := range []configv1.PlatformType{configv1.AWSPlatformType, configv1.NonePlatformType, "unrecognized"} { + t.Run(string(platform), func(t *testing.T) { + + cc := newControllerConfig(ctrlcommon.ControllerConfigName, platform) + mcp := helpers.NewMachineConfigPool("master", nil, helpers.MasterSelector, "v0") + mcp2 := helpers.NewMachineConfigPool("worker", nil, helpers.WorkerSelector, "v0") + mcps := []*mcfgv1.MachineConfigPool{mcp, mcp2} + + features := &osev1.FeatureGate{ + ObjectMeta: metav1.ObjectMeta{ + Name: ctrlcommon.ClusterFeatureInstanceName, + }, + Spec: osev1.FeatureGateSpec{ + FeatureGateSelection: osev1.FeatureGateSelection{ + FeatureSet: osev1.CustomNoUpgrade, + CustomNoUpgrade: &osev1.CustomFeatureGates{ + Enabled: []string{"CSIMigration"}, + }, + }, + }, + } + + mcs, err := RunFeatureGateBootstrap("../../../templates", features, cc, mcps) + if err != nil { + t.Errorf("could not run feature gate bootstrap: %v", err) + } + if len(mcs) != 2 { + t.Errorf("expected 2 machine configs generated with the custom feature gate, got %d configs", len(mcs)) + } + + for _, mc := range mcs { + ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw) + regfile := ignCfg.Storage.Files[0] + conf, err := dataurl.DecodeString(*regfile.Contents.Source) + require.NoError(t, err) + + originalKubeConfig, _ := decodeKubeletConfig(conf.Data) + defaultFeatureGates, err := generateFeatureMap(createNewDefaultFeatureGate()) + if err != nil { + t.Errorf("could not generate defaultFeatureGates: %v", err) + } + if reflect.DeepEqual(originalKubeConfig.FeatureGates, *defaultFeatureGates) { + t.Errorf("template FeatureGates should not match default openshift/api FeatureGates: (default=%v)", defaultFeatureGates) + } + if !originalKubeConfig.FeatureGates["CSIMigration"] { + t.Errorf("template FeatureGates should contain CSIMigration: %v", originalKubeConfig.FeatureGates) + } + } + }) + } +} diff --git a/pkg/controller/template/template_controller.go b/pkg/controller/template/template_controller.go index 909224bd03..96fa61e296 100644 --- a/pkg/controller/template/template_controller.go +++ b/pkg/controller/template/template_controller.go @@ -523,6 +523,6 @@ func getMachineConfigsForControllerConfig(templatesDir string, config *mcfgv1.Co } // RunBootstrap runs the tempate controller in boostrap mode. -func RunBootstrap(templatesDir string, config *mcfgv1.ControllerConfig, pullSecretRaw []byte) ([]*mcfgv1.MachineConfig, error) { - return getMachineConfigsForControllerConfig(templatesDir, config, pullSecretRaw, nil) +func RunBootstrap(templatesDir string, config *mcfgv1.ControllerConfig, pullSecretRaw []byte, featureGate *configv1.FeatureGate) ([]*mcfgv1.MachineConfig, error) { + return getMachineConfigsForControllerConfig(templatesDir, config, pullSecretRaw, featureGate) }