diff --git a/go.mod b/go.mod index d557089846..eccf3cd0b6 100644 --- a/go.mod +++ b/go.mod @@ -190,7 +190,7 @@ replace ( k8s.io/api => k8s.io/api v0.28.3 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.3 k8s.io/apimachinery => k8s.io/apimachinery v0.28.3 - k8s.io/apiserver => github.com/openshift/kubernetes-apiserver v0.0.0-20240112142943-d5600c13f38a + k8s.io/apiserver => github.com/openshift/kubernetes-apiserver v0.0.0-20260211130448-27fe2869e6c8 // points to openshift-apiserver-4.15-kubernetes-1.28.3 k8s.io/cli-runtime => k8s.io/cli-runtime v0.28.3 k8s.io/client-go => k8s.io/client-go v0.28.3 k8s.io/cloud-provider => k8s.io/cloud-provider v0.28.3 diff --git a/go.sum b/go.sum index 197665fa2e..fdb26f93b3 100644 --- a/go.sum +++ b/go.sum @@ -343,8 +343,8 @@ github.com/openshift/docker-distribution/v3 v3.0.0-20240216102618-dbaf4de30867 h github.com/openshift/docker-distribution/v3 v3.0.0-20240216102618-dbaf4de30867/go.mod h1:+fqBJ4vPYo4Uu1ZE4d+bUtTLRXfdSL3NvCZIZ9GHv58= github.com/openshift/kubernetes v0.0.0-20231102175044-c01bd450743e h1:VQAVi9Vw2OaxXJtQd/uB3qozjxh19cZSjA4Phnf/kSM= github.com/openshift/kubernetes v0.0.0-20231102175044-c01bd450743e/go.mod h1:NhAysZWvHtNcJFFHic87ofxQN7loylCQwg3ZvXVDbag= -github.com/openshift/kubernetes-apiserver v0.0.0-20240112142943-d5600c13f38a h1:M9I+ouvdUpNBV9+uwOSFhvB1dzmH7mmQkLOf9s4TT5M= -github.com/openshift/kubernetes-apiserver v0.0.0-20240112142943-d5600c13f38a/go.mod h1:YIpM+9wngNAv8Ctt0rHG4vQuX/I5rvkEMtZtsxW2rNM= +github.com/openshift/kubernetes-apiserver v0.0.0-20260211130448-27fe2869e6c8 h1:Eukt0aIYNvvUmiH30V8u7YVO+2z8/Ws4ORe8PMU3Ubc= +github.com/openshift/kubernetes-apiserver v0.0.0-20260211130448-27fe2869e6c8/go.mod h1:YIpM+9wngNAv8Ctt0rHG4vQuX/I5rvkEMtZtsxW2rNM= github.com/openshift/library-go v0.0.0-20230808150704-ce4395c85e8c h1:UJjxHFSTcasHxRXtDc3od9p7UJUBJxUKjhZHFyp2uUQ= github.com/openshift/library-go v0.0.0-20230808150704-ce4395c85e8c/go.mod h1:ZFwNwC3opc/7aOvzUbU95zp33Lbxet48h80ryH3p6DY= github.com/openshift/moby-moby v0.0.0-20190308215630-da810a85109d h1:fLITXDjxMSvUDjnXs/zljIWktbST9+Om8XbrmmM7T4I= diff --git a/vendor/k8s.io/apiserver/pkg/server/options/client-go-certutil-copy.go b/vendor/k8s.io/apiserver/pkg/server/options/client-go-certutil-copy.go new file mode 100644 index 0000000000..acc96ccb67 --- /dev/null +++ b/vendor/k8s.io/apiserver/pkg/server/options/client-go-certutil-copy.go @@ -0,0 +1,202 @@ +/* +Copyright 2018 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// NOTE: As part of the work to resolve https://issues.redhat.com/browse/OCPBUGS-61760 +// and https://issues.redhat.com/browse/OCPBUGS-61759 it was decided that copying +// the loopback certification creation utility function to a local +// utility function would save significant effort over cherry-picking +// the fix from https://github.com/kubernetes/kubernetes/pull/130047 to +// https://github.com/openshift/kubernetes-client-go . +// We don't expect to backport many changes on top of this and it seemed +// lower risk than switching all of our aggregated API server to our fork of +// client-go because almost nothing currently depends on it. +// +// The only difference we expect to be present when comparing the +// `GenerateSelfSignedCertKeyWithFixtures` function in this file +// with the one present in the v0.28.3 client-go source[1] is +// the modification to change the maxAge variable to ~3 years. +// +// [1]: https://github.com/kubernetes/client-go/blob/9d454d6c3adfb3beb32b38eb692c1688df56c0ec/util/cert/cert.go#L93-L230 + +package options + +import ( + "bytes" + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math" + "math/big" + "net" + "os" + "path/filepath" + "strings" + "time" + + "k8s.io/client-go/util/keyutil" + netutils "k8s.io/utils/net" +) + +// GenerateSelfSignedCertKey creates a self-signed certificate and key for the given host. +// Host may be an IP or a DNS name +// You may also specify additional subject alt names (either ip or dns names) for the certificate. +func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS []string) ([]byte, []byte, error) { + return GenerateSelfSignedCertKeyWithFixtures(host, alternateIPs, alternateDNS, "") +} + +// GenerateSelfSignedCertKeyWithFixtures creates a self-signed certificate and key for the given host. +// Host may be an IP or a DNS name. You may also specify additional subject alt names (either ip or dns names) +// for the certificate. +// +// If fixtureDirectory is non-empty, it is a directory path which can contain pre-generated certs. The format is: +// _-_-.crt +// _-_-.key +// Certs/keys not existing in that directory are created. +func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, alternateDNS []string, fixtureDirectory string) ([]byte, []byte, error) { + validFrom := time.Now().Add(-time.Hour) // valid an hour earlier to avoid flakes due to clock skew + + // NOTE: Modified from the original of 1 year to 3 years to fix + // - https://issues.redhat.com/browse/OCPBUGS-61760 + // - https://issues.redhat.com/browse/OCPBUGS-61759 + // Achieves the same result as the upstream change in https://github.com/kubernetes/kubernetes/pull/130047 + maxAge := time.Hour * 24 * (3*365 + 1) // three year self-signed certs + + baseName := fmt.Sprintf("%s_%s_%s", host, strings.Join(ipsToStrings(alternateIPs), "-"), strings.Join(alternateDNS, "-")) + certFixturePath := filepath.Join(fixtureDirectory, baseName+".crt") + keyFixturePath := filepath.Join(fixtureDirectory, baseName+".key") + if len(fixtureDirectory) > 0 { + cert, err := os.ReadFile(certFixturePath) + if err == nil { + key, err := os.ReadFile(keyFixturePath) + if err == nil { + return cert, key, nil + } + return nil, nil, fmt.Errorf("cert %s can be read, but key %s cannot: %v", certFixturePath, keyFixturePath, err) + } + maxAge = 100 * time.Hour * 24 * 365 // 100 years fixtures + } + + caKey, err := rsa.GenerateKey(cryptorand.Reader, 2048) + if err != nil { + return nil, nil, err + } + // returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max). + serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64-1)) + if err != nil { + return nil, nil, err + } + serial = new(big.Int).Add(serial, big.NewInt(1)) + caTemplate := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: fmt.Sprintf("%s-ca@%d", host, time.Now().Unix()), + }, + NotBefore: validFrom, + NotAfter: validFrom.Add(maxAge), + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + + caDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey) + if err != nil { + return nil, nil, err + } + + caCertificate, err := x509.ParseCertificate(caDERBytes) + if err != nil { + return nil, nil, err + } + + priv, err := rsa.GenerateKey(cryptorand.Reader, 2048) + if err != nil { + return nil, nil, err + } + // returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max). + serial, err = cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64-1)) + if err != nil { + return nil, nil, err + } + serial = new(big.Int).Add(serial, big.NewInt(1)) + template := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()), + }, + NotBefore: validFrom, + NotAfter: validFrom.Add(maxAge), + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + if ip := netutils.ParseIPSloppy(host); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, host) + } + + template.IPAddresses = append(template.IPAddresses, alternateIPs...) + template.DNSNames = append(template.DNSNames, alternateDNS...) + + derBytes, err := x509.CreateCertificate(cryptorand.Reader, &template, caCertificate, &priv.PublicKey, caKey) + if err != nil { + return nil, nil, err + } + + // Generate cert, followed by ca + certBuffer := bytes.Buffer{} + if err := pem.Encode(&certBuffer, &pem.Block{Type: CertificateBlockType, Bytes: derBytes}); err != nil { + return nil, nil, err + } + if err := pem.Encode(&certBuffer, &pem.Block{Type: CertificateBlockType, Bytes: caDERBytes}); err != nil { + return nil, nil, err + } + + // Generate key + keyBuffer := bytes.Buffer{} + if err := pem.Encode(&keyBuffer, &pem.Block{Type: keyutil.RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { + return nil, nil, err + } + + if len(fixtureDirectory) > 0 { + if err := os.WriteFile(certFixturePath, certBuffer.Bytes(), 0644); err != nil { + return nil, nil, fmt.Errorf("failed to write cert fixture to %s: %v", certFixturePath, err) + } + if err := os.WriteFile(keyFixturePath, keyBuffer.Bytes(), 0600); err != nil { + return nil, nil, fmt.Errorf("failed to write key fixture to %s: %v", certFixturePath, err) + } + } + + return certBuffer.Bytes(), keyBuffer.Bytes(), nil +} + +func ipsToStrings(ips []net.IP) []string { + ss := make([]string, 0, len(ips)) + for _, ip := range ips { + ss = append(ss, ip.String()) + } + return ss +} + +const ( + // CertificateBlockType is a possible value for pem.Block.Type. + CertificateBlockType = "CERTIFICATE" + // CertificateRequestBlockType is a possible value for pem.Block.Type. + CertificateRequestBlockType = "CERTIFICATE REQUEST" +) diff --git a/vendor/k8s.io/apiserver/pkg/server/options/feature.go b/vendor/k8s.io/apiserver/pkg/server/options/feature.go index 35596fba69..87b4aaa2e1 100644 --- a/vendor/k8s.io/apiserver/pkg/server/options/feature.go +++ b/vendor/k8s.io/apiserver/pkg/server/options/feature.go @@ -17,16 +17,24 @@ limitations under the License. package options import ( + "fmt" + "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/util/feature" + utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" ) type FeatureOptions struct { EnableProfiling bool DebugSocketPath string EnableContentionProfiling bool + EnablePriorityAndFairness bool } func NewFeatureOptions() *FeatureOptions { @@ -36,6 +44,7 @@ func NewFeatureOptions() *FeatureOptions { EnableProfiling: defaults.EnableProfiling, DebugSocketPath: defaults.DebugSocketPath, EnableContentionProfiling: defaults.EnableContentionProfiling, + EnablePriorityAndFairness: true, } } @@ -50,9 +59,11 @@ func (o *FeatureOptions) AddFlags(fs *pflag.FlagSet) { "Enable block profiling, if profiling is enabled") fs.StringVar(&o.DebugSocketPath, "debug-socket-path", o.DebugSocketPath, "Use an unprotected (no authn/authz) unix-domain socket for profiling with the given path") + fs.BoolVar(&o.EnablePriorityAndFairness, "enable-priority-and-fairness", o.EnablePriorityAndFairness, ""+ + "If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness") } -func (o *FeatureOptions) ApplyTo(c *server.Config) error { +func (o *FeatureOptions) ApplyTo(c *server.Config, clientset kubernetes.Interface, informers informers.SharedInformerFactory) error { if o == nil { return nil } @@ -61,6 +72,19 @@ func (o *FeatureOptions) ApplyTo(c *server.Config) error { c.DebugSocketPath = o.DebugSocketPath c.EnableContentionProfiling = o.EnableContentionProfiling + if o.EnablePriorityAndFairness && feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) { + if c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight <= 0 { + return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight) + + } + c.FlowControl = utilflowcontrol.New( + informers, + clientset.FlowcontrolV1beta3(), + c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight, + c.RequestTimeout/4, + ) + } + return nil } diff --git a/vendor/k8s.io/apiserver/pkg/server/options/recommended.go b/vendor/k8s.io/apiserver/pkg/server/options/recommended.go index 69f8fb5155..eb7e67b367 100644 --- a/vendor/k8s.io/apiserver/pkg/server/options/recommended.go +++ b/vendor/k8s.io/apiserver/pkg/server/options/recommended.go @@ -17,20 +17,15 @@ limitations under the License. package options import ( - "fmt" - "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/util/feature" - utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/component-base/featuregate" - "k8s.io/klog/v2" ) // RecommendedOptions contains the recommended options for running an API server. @@ -122,17 +117,17 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { if err := o.Audit.ApplyTo(&config.Config); err != nil { return err } - if err := o.Features.ApplyTo(&config.Config); err != nil { - return err - } if err := o.CoreAPI.ApplyTo(config); err != nil { return err } - initializers, err := o.ExtraAdmissionInitializers(config) + kubeClient, err := kubernetes.NewForConfig(config.ClientConfig) if err != nil { return err } - kubeClient, err := kubernetes.NewForConfig(config.ClientConfig) + if err := o.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil { + return err + } + initializers, err := o.ExtraAdmissionInitializers(config) if err != nil { return err } @@ -144,22 +139,6 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { initializers...); err != nil { return err } - if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) { - if config.ClientConfig != nil { - if config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight <= 0 { - return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", config.MaxRequestsInFlight, config.MaxMutatingRequestsInFlight) - - } - config.FlowControl = utilflowcontrol.New( - config.SharedInformerFactory, - kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1beta3(), - config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight, - config.RequestTimeout/4, - ) - } else { - klog.Warningf("Neither kubeconfig is provided nor service-account is mounted, so APIPriorityAndFairness will be disabled") - } - } return nil } diff --git a/vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go b/vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go index f9d574d5d2..1373d8a4d7 100644 --- a/vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go +++ b/vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go @@ -62,8 +62,7 @@ type ServerRunOptions struct { // decoded in a write request. 0 means no limit. // We intentionally did not add a flag for this option. Users of the // apiserver library can wire it to a flag. - MaxRequestBodyBytes int64 - EnablePriorityAndFairness bool + MaxRequestBodyBytes int64 // ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP // Server during the graceful termination of the apiserver. If true, we wait @@ -104,7 +103,6 @@ func NewServerRunOptions() *ServerRunOptions { ShutdownWatchTerminationGracePeriod: defaults.ShutdownWatchTerminationGracePeriod, JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes, MaxRequestBodyBytes: defaults.MaxRequestBodyBytes, - EnablePriorityAndFairness: true, ShutdownSendRetryAfter: false, } } @@ -325,9 +323,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { "handler, which picks a randomized value above this number as the connection timeout, "+ "to spread out load.") - fs.BoolVar(&s.EnablePriorityAndFairness, "enable-priority-and-fairness", s.EnablePriorityAndFairness, ""+ - "If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness") - fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+ "Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+ "will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+ diff --git a/vendor/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go b/vendor/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go index 2317be82d2..b18a750ba4 100644 --- a/vendor/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go +++ b/vendor/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go @@ -24,7 +24,6 @@ import ( "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/dynamiccertificates" "k8s.io/client-go/rest" - certutil "k8s.io/client-go/util/cert" ) type SecureServingOptionsWithLoopback struct { @@ -51,7 +50,7 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.Se // create self-signed cert+key with the fake server.LoopbackClientServerNameOverride and // let the server return it when the loopback client connects. - certPem, keyPem, err := certutil.GenerateSelfSignedCertKey(server.LoopbackClientServerNameOverride, nil, nil) + certPem, keyPem, err := GenerateSelfSignedCertKey(server.LoopbackClientServerNameOverride, nil, nil) // use forked GenererateSelfSignedCertKey with extended expiry if err != nil { return fmt.Errorf("failed to generate self-signed certificate for loopback connection: %v", err) } diff --git a/vendor/modules.txt b/vendor/modules.txt index e975bf005b..e271a33992 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1282,7 +1282,7 @@ k8s.io/apimachinery/pkg/watch k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/netutil k8s.io/apimachinery/third_party/forked/golang/reflect -# k8s.io/apiserver v0.28.3 => github.com/openshift/kubernetes-apiserver v0.0.0-20240112142943-d5600c13f38a +# k8s.io/apiserver v0.28.3 => github.com/openshift/kubernetes-apiserver v0.0.0-20260211130448-27fe2869e6c8 ## explicit; go 1.20 k8s.io/apiserver/pkg/admission k8s.io/apiserver/pkg/admission/cel @@ -2271,7 +2271,7 @@ sigs.k8s.io/yaml # k8s.io/api => k8s.io/api v0.28.3 # k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.3 # k8s.io/apimachinery => k8s.io/apimachinery v0.28.3 -# k8s.io/apiserver => github.com/openshift/kubernetes-apiserver v0.0.0-20240112142943-d5600c13f38a +# k8s.io/apiserver => github.com/openshift/kubernetes-apiserver v0.0.0-20260211130448-27fe2869e6c8 # k8s.io/cli-runtime => k8s.io/cli-runtime v0.28.3 # k8s.io/client-go => k8s.io/client-go v0.28.3 # k8s.io/cloud-provider => k8s.io/cloud-provider v0.28.3