Skip to content

CNTRLPLANE-2589: update cases to execute as parallel #833

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ropatil010 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

CNTRLPLANE-2589: update cases to execute as parallel #833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,13 @@ make build

```bash
# Run a specific test suite
./cluster-authentication-operator-tests-ext run-suite openshift/cluster-authentication-operator/operator/serial
./cluster-authentication-operator-tests-ext run-suite openshift/cluster-authentication-operator/operator/parallel

# Run with serial execution (1 worker)
./cluster-authentication-operator-tests-ext run-suite openshift/cluster-authentication-operator/operator/serial -c 1
# Run with parallel execution (4 workers)
./cluster-authentication-operator-tests-ext run-suite openshift/cluster-authentication-operator/operator/parallel -c 4

# Run with JUnit output
./cluster-authentication-operator-tests-ext run-suite openshift/cluster-authentication-operator/operator/serial --junit-path "${ARTIFACT_DIR}/junit.xml"
./cluster-authentication-operator-tests-ext run-suite openshift/cluster-authentication-operator/operator/parallel --junit-path "${ARTIFACT_DIR}/junit.xml"

# Run a specific test
./cluster-authentication-operator-tests-ext run-test "test-name"
Expand All @@ -54,7 +54,7 @@ make build
./cluster-authentication-operator-tests-ext list suites

# List tests in a suite
./cluster-authentication-operator-tests-ext list tests --suite=openshift/cluster-authentication-operator/operator/serial
./cluster-authentication-operator-tests-ext list tests --suite=openshift/cluster-authentication-operator/operator/parallel
```

For more information about the OTE framework, see the [openshift-tests-extension documentation](https://github.com/openshift-eng/openshift-tests-extension).
Expand Down
11 changes: 11 additions & 0 deletions cmd/cluster-authentication-operator-tests-ext/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,17 @@ func prepareOperatorTestsRegistry() (*oteextension.Registry, error) {

// The following suite runs tests that verify the operator's behaviour.
// This suite is executed only on pull requests targeting this repository.
// Tests tagged with [Parallel] and any of [Operator], [OIDC], [Templates], [Tokens] are included in this suite.
extension.AddSuite(oteextension.Suite{
Name: "openshift/cluster-authentication-operator/operator/parallel",
Parallelism: 4,
Qualifiers: []string{
`name.contains("[Parallel]") && (name.contains("[Operator]") || name.contains("[OIDC]") || name.contains("[Templates]") || name.contains("[Tokens]"))`,
},
})

// The following suite runs tests that must execute serially (one at a time)
// because they modify cluster-wide resources like OAuth configuration.
// Tests tagged with [Serial] and any of [Operator], [OIDC], [Templates], [Tokens] are included in this suite.
extension.AddSuite(oteextension.Suite{
Name: "openshift/cluster-authentication-operator/operator/serial",
Expand Down
42 changes: 11 additions & 31 deletions test/e2e/certs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"bytes"
"context"
"encoding/pem"
"strings"
"testing"
"time"

Expand All @@ -15,35 +14,26 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/cert"
"k8s.io/client-go/util/keyutil"

operatorv1 "github.com/openshift/api/operator/v1"
configclient "github.com/openshift/client-go/config/clientset/versioned"
operatorclient "github.com/openshift/client-go/operator/clientset/versioned"
"github.com/openshift/library-go/pkg/operator/v1helpers"

e2e "github.com/openshift/cluster-authentication-operator/test/library"
)

var _ = g.Describe("[sig-auth] authentication operator", func() {
g.It("[Operator][Certs][Serial] TestRouterCerts", func() {
g.It("[Operator][Certs][Parallel] TestRouterCerts", func() {
testRouterCerts(g.GinkgoTB())
})
})

func testRouterCerts(t testing.TB) {
kubeConfig := e2e.NewClientConfigForTest(t)
clients := e2e.NewTestClients(t)

kubeClient, err := kubernetes.NewForConfig(kubeConfig)
require.NoError(t, err)
configClient, err := configclient.NewForConfig(kubeConfig)
require.NoError(t, err)
operatorClientset, err := operatorclient.NewForConfig(kubeConfig)
require.NoError(t, err)
// make sure cluster operator is settled before continuing
err = e2e.WaitForClusterOperatorAvailableNotProgressingNotDegraded(t, configClient.ConfigV1(), "authentication")
err := e2e.WaitForClusterOperatorAvailableNotProgressingNotDegraded(t, clients.ConfigClient.ConfigV1(), "authentication")
require.NoError(t, err)

// generate crypto materials
Expand All @@ -59,19 +49,9 @@ func testRouterCerts(t testing.TB) {
privateKey, err := keyutil.MarshalPrivateKeyToPEM(server.PrivateKey)
require.NoError(t, err)
// Generate a valid Kubernetes name from the test name
// Replace invalid characters with hyphens and ensure it starts/ends with alphanumeric
secretName := strings.ToLower(t.Name())
secretName = strings.ReplaceAll(secretName, "/", "-")
secretName = strings.ReplaceAll(secretName, " ", "-")
secretName = strings.ReplaceAll(secretName, "[", "")
secretName = strings.ReplaceAll(secretName, "]", "")
secretName = strings.Trim(secretName, "-")
if len(secretName) > 63 {
secretName = secretName[:63]
}
secretName = strings.TrimRight(secretName, "-")
secretName := e2e.SanitizeResourceName(t.Name())

secret, err := kubeClient.CoreV1().Secrets("openshift-ingress").Create(
secret, err := clients.KubeClient.CoreV1().Secrets("openshift-ingress").Create(
context.TODO(),
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{GenerateName: secretName + "-"},
Expand All @@ -83,27 +63,27 @@ func testRouterCerts(t testing.TB) {
}, metav1.CreateOptions{})
require.NoError(t, err)
defer func() {
_ = kubeClient.CoreV1().Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, metav1.DeleteOptions{})
_ = clients.KubeClient.CoreV1().Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, metav1.DeleteOptions{})
}()

// set custom ingress defaultCertificate
ingressController, err := operatorClientset.OperatorV1().IngressControllers("openshift-ingress-operator").Get(context.TODO(), "default", metav1.GetOptions{})
ingressController, err := clients.OperatorClient.OperatorV1().IngressControllers("openshift-ingress-operator").Get(context.TODO(), "default", metav1.GetOptions{})
require.NoError(t, err)
backup := ingressController.Spec.DefaultCertificate
defer func() {
ingressController, err := operatorClientset.OperatorV1().IngressControllers("openshift-ingress-operator").Get(context.TODO(), "default", metav1.GetOptions{})
ingressController, err := clients.OperatorClient.OperatorV1().IngressControllers("openshift-ingress-operator").Get(context.TODO(), "default", metav1.GetOptions{})
require.NoError(t, err)
ingressController.Spec.DefaultCertificate = backup
_, _ = operatorClientset.OperatorV1().IngressControllers(ingressController.Namespace).Update(context.TODO(), ingressController, metav1.UpdateOptions{})
_, _ = clients.OperatorClient.OperatorV1().IngressControllers(ingressController.Namespace).Update(context.TODO(), ingressController, metav1.UpdateOptions{})
}()
ingressController.Spec.DefaultCertificate = &corev1.LocalObjectReference{Name: secret.Name}
_, err = operatorClientset.OperatorV1().IngressControllers(ingressController.Namespace).Update(context.TODO(), ingressController, metav1.UpdateOptions{})
_, err = clients.OperatorClient.OperatorV1().IngressControllers(ingressController.Namespace).Update(context.TODO(), ingressController, metav1.UpdateOptions{})
require.NoError(t, err)

// wait for RouterCertsDegraded == true
var condition *operatorv1.OperatorCondition
err = wait.PollImmediate(time.Second, 10*time.Minute, func() (bool, error) {
config, err := operatorClientset.OperatorV1().Authentications().Get(context.TODO(), "cluster", metav1.GetOptions{})
config, err := clients.OperatorClient.OperatorV1().Authentications().Get(context.TODO(), "cluster", metav1.GetOptions{})
if errors.IsNotFound(err) {
t.Logf("Unable to retrieve operator config: %v", err)
return false, nil
Expand Down
79 changes: 26 additions & 53 deletions test/e2e/custom_route.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package e2e

import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"net/http"
Expand All @@ -23,33 +22,26 @@ import (
"k8s.io/client-go/util/keyutil"

configv1 "github.com/openshift/api/config/v1"
configclient "github.com/openshift/client-go/config/clientset/versioned"
routeclient "github.com/openshift/client-go/route/clientset/versioned"
configclient "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
routeclient "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1"

e2e "github.com/openshift/cluster-authentication-operator/test/library"
)

var _ = g.Describe("[sig-auth] authentication operator", func() {
g.It("[Operator][Routes][Serial] TestCustomRouterCerts", func() {
g.It("[Operator][Routes][Parallel] TestCustomRouterCerts", func() {
testCustomRouterCerts(g.GinkgoTB())
})
})

func testCustomRouterCerts(t testing.TB) {
kubeConfig := e2e.NewClientConfigForTest(t)

kubeClient, err := kubernetes.NewForConfig(kubeConfig)
require.NoError(t, err)
configClient, err := configclient.NewForConfig(kubeConfig)
require.NoError(t, err)
routeClient, err := routeclient.NewForConfig(kubeConfig)
require.NoError(t, err)
clients := e2e.NewTestClients(t)

// generate crypto materials
rootCA := e2e.NewCertificateAuthorityCertificate(t, nil)
intermediateCA := e2e.NewCertificateAuthorityCertificate(t, rootCA)
// check that the route is set to defaults if a non-existant secret is provided
ingressConfig, err := configClient.ConfigV1().Ingresses().Get(context.TODO(), "cluster", metav1.GetOptions{})
ingressConfig, err := clients.ConfigClient.ConfigV1().Ingresses().Get(context.TODO(), "cluster", metav1.GetOptions{})
require.NoError(t, err)

fooHostname := "foo." + ingressConfig.Spec.Domain
Expand All @@ -62,19 +54,9 @@ func testCustomRouterCerts(t testing.TB) {
customServerCertPEM := pem.EncodeToMemory(&pem.Block{Type: cert.CertificateBlockType, Bytes: server.Certificate.Raw})

// Generate a valid Kubernetes name from the test name
// Replace invalid characters with hyphens and ensure it starts/ends with alphanumeric
secretName := strings.ToLower(t.Name())
secretName = strings.ReplaceAll(secretName, "/", "-")
secretName = strings.ReplaceAll(secretName, " ", "-")
secretName = strings.ReplaceAll(secretName, "[", "")
secretName = strings.ReplaceAll(secretName, "]", "")
secretName = strings.Trim(secretName, "-")
if len(secretName) > 63 {
secretName = secretName[:63]
}
secretName = strings.TrimRight(secretName, "-")
secretName := e2e.SanitizeResourceName(t.Name())

secret, err := kubeClient.CoreV1().Secrets("openshift-config").Create(
secret, err := clients.KubeClient.CoreV1().Secrets("openshift-config").Create(
context.TODO(),
&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{GenerateName: secretName + "-"},
Expand All @@ -86,33 +68,33 @@ func testCustomRouterCerts(t testing.TB) {
}, metav1.CreateOptions{})
require.NoError(t, err)
defer func() {
err = removeComponentRoute(t, configClient, "openshift-authentication", "oauth-openshift")
err = removeComponentRoute(t, clients.ConfigClient.ConfigV1(), "openshift-authentication", "oauth-openshift")
require.NoError(t, err)
err = kubeClient.CoreV1().Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, metav1.DeleteOptions{})
err = clients.KubeClient.CoreV1().Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, metav1.DeleteOptions{})
require.NoError(t, err)
}()

// check that the trust-distribution works by publishing the server certificate
distributedServerCert, err := kubeClient.CoreV1().ConfigMaps("openshift-config-managed").Get(context.Background(), "oauth-serving-cert", metav1.GetOptions{})
distributedServerCert, err := clients.KubeClient.CoreV1().ConfigMaps("openshift-config-managed").Get(context.Background(), "oauth-serving-cert", metav1.GetOptions{})
require.NoError(t, err)

distributedServerCertPem := distributedServerCert.Data["ca-bundle.crt"]
require.NotZero(t, len(distributedServerCertPem))

// set a custom hostname without a secret
err = getAndUpdateComponentRoute(t, configClient, &configv1.ComponentRouteSpec{
err = getAndUpdateComponentRoute(t, clients.ConfigClient.ConfigV1(), &configv1.ComponentRouteSpec{
Namespace: "openshift-authentication",
Name: "oauth-openshift",
Hostname: "foo.bar.com",
})
require.NoError(t, err)

// check that the hostname was updated
err = checkRouteHostname(t, routeClient, "openshift-authentication", "oauth-openshift", "foo.bar.com")
err = checkRouteHostname(t, clients.RouteClient.RouteV1(), "openshift-authentication", "oauth-openshift", "foo.bar.com")
require.NoError(t, err)

// update the hostname and provide a custom secret that does not exist
err = getAndUpdateComponentRoute(t, configClient, &configv1.ComponentRouteSpec{
err = getAndUpdateComponentRoute(t, clients.ConfigClient.ConfigV1(), &configv1.ComponentRouteSpec{
Namespace: "openshift-authentication",
Name: "oauth-openshift",
Hostname: "new.foo.bar.com",
Expand All @@ -123,11 +105,11 @@ func testCustomRouterCerts(t testing.TB) {
require.NoError(t, err)

// check that the hostname of the route is not changed because a missing secret was provided
err = checkRouteHostname(t, routeClient, "openshift-authentication", "oauth-openshift", "foo.bar.com")
err = checkRouteHostname(t, clients.RouteClient.RouteV1(), "openshift-authentication", "oauth-openshift", "foo.bar.com")
require.NoError(t, err)

// Update the hostname and use a valid secret
err = getAndUpdateComponentRoute(t, configClient, &configv1.ComponentRouteSpec{
err = getAndUpdateComponentRoute(t, clients.ConfigClient.ConfigV1(), &configv1.ComponentRouteSpec{
Namespace: "openshift-authentication",
Name: "oauth-openshift",
Hostname: configv1.Hostname(fooHostname),
Expand All @@ -137,9 +119,9 @@ func testCustomRouterCerts(t testing.TB) {
})
require.NoError(t, err)

waitForDistributedCert(t, kubeClient, customServerCertPEM)
waitForDistributedCert(t, clients.KubeClient, customServerCertPEM)

err = checkRouteHostname(t, routeClient, "openshift-authentication", "oauth-openshift", fooHostname)
err = checkRouteHostname(t, clients.RouteClient.RouteV1(), "openshift-authentication", "oauth-openshift", fooHostname)
require.NoError(t, err)

// Check that the route is serving
Expand All @@ -165,16 +147,7 @@ func waitForDistributedCert(t testing.TB, kubeClient kubernetes.Interface, expec
}

func pollForCustomServingCertificates(t testing.TB, hostname string, certificate *x509.Certificate) error {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
httpClient := &http.Client{
Timeout: 5 * time.Second,
Transport: transport,
}
httpClient := e2e.NewInsecureHTTPClient(5 * time.Second)
req, err := http.NewRequest(http.MethodGet, hostname, nil)
if err != nil {
return err
Expand Down Expand Up @@ -206,9 +179,9 @@ func pollForCustomServingCertificates(t testing.TB, hostname string, certificate
})
}

func getAndUpdateComponentRoute(t testing.TB, configClient *configclient.Clientset, componentRoute *configv1.ComponentRouteSpec) error {
func getAndUpdateComponentRoute(t testing.TB, configClient configclient.ConfigV1Interface, componentRoute *configv1.ComponentRouteSpec) error {
return wait.PollImmediate(time.Second, time.Minute, func() (bool, error) {
ingressConfig, err := configClient.ConfigV1().Ingresses().Get(context.TODO(), "cluster", metav1.GetOptions{})
ingressConfig, err := configClient.Ingresses().Get(context.TODO(), "cluster", metav1.GetOptions{})
if errors.IsNotFound(err) {
t.Logf("Unable to retrieve ingress config: %v", err)
return false, nil
Expand All @@ -229,17 +202,17 @@ func getAndUpdateComponentRoute(t testing.TB, configClient *configclient.Clients
ingressConfig.Spec.ComponentRoutes = append(ingressConfig.Spec.ComponentRoutes, *componentRoute)
}

ingressConfig, err = configClient.ConfigV1().Ingresses().Update(context.TODO(), ingressConfig, metav1.UpdateOptions{})
ingressConfig, err = configClient.Ingresses().Update(context.TODO(), ingressConfig, metav1.UpdateOptions{})
if err != nil {
return false, nil
}
return true, nil
})
}

func checkRouteHostname(t testing.TB, routeClient *routeclient.Clientset, routeNamespace string, routeName string, hostname string) error {
func checkRouteHostname(t testing.TB, routeClient routeclient.RouteV1Interface, routeNamespace string, routeName string, hostname string) error {
return wait.PollImmediate(time.Second, time.Minute, func() (bool, error) {
route, err := routeClient.RouteV1().Routes(routeNamespace).Get(context.TODO(), routeName, metav1.GetOptions{})
route, err := routeClient.Routes(routeNamespace).Get(context.TODO(), routeName, metav1.GetOptions{})
if errors.IsNotFound(err) {
t.Logf("Unable to retrieve route: %v", err)
return false, nil
Expand All @@ -252,9 +225,9 @@ func checkRouteHostname(t testing.TB, routeClient *routeclient.Clientset, routeN
})
}

func removeComponentRoute(t testing.TB, configClient *configclient.Clientset, namespace string, name string) error {
func removeComponentRoute(t testing.TB, configClient configclient.ConfigV1Interface, namespace string, name string) error {
return wait.PollImmediate(time.Second, time.Minute, func() (bool, error) {
ingressConfig, err := configClient.ConfigV1().Ingresses().Get(context.TODO(), "cluster", metav1.GetOptions{})
ingressConfig, err := configClient.Ingresses().Get(context.TODO(), "cluster", metav1.GetOptions{})
if errors.IsNotFound(err) {
t.Logf("Unable to retrieve ingress config: %v", err)
return false, nil
Expand All @@ -270,7 +243,7 @@ func removeComponentRoute(t testing.TB, configClient *configclient.Clientset, na
ingressConfig.Spec.ComponentRoutes = append(ingressConfig.Spec.ComponentRoutes[:i], ingressConfig.Spec.ComponentRoutes[i+1:]...)

// update the ingress resource
_, err = configClient.ConfigV1().Ingresses().Update(context.TODO(), ingressConfig, metav1.UpdateOptions{})
_, err = configClient.Ingresses().Update(context.TODO(), ingressConfig, metav1.UpdateOptions{})
if err != nil {
return false, nil
}
Expand Down
10 changes: 3 additions & 7 deletions test/e2e/gitlab.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,31 +6,27 @@ import (
g "github.com/onsi/ginkgo/v2"
"github.com/stretchr/testify/require"

"k8s.io/client-go/kubernetes"

test "github.com/openshift/cluster-authentication-operator/test/library"
)

var _ = g.Describe("[sig-auth] authentication operator", func() {
g.It("[OIDC][Serial] TestGitLabAsOIDCPasswordGrantCheck", func() {
g.It("[OIDC][Parallel] TestGitLabAsOIDCPasswordGrantCheck", func() {
testGitLabAsOIDCPasswordGrantCheck(g.GinkgoTB())
})
})

func testGitLabAsOIDCPasswordGrantCheck(t testing.TB) {
clients := test.NewTestClients(t)
kubeConfig := test.NewClientConfigForTest(t)

kubeClients, err := kubernetes.NewForConfig(kubeConfig)
require.NoError(t, err)

_, idpName, cleanups := test.AddGitlabIDP(t, kubeConfig)
defer test.IDPCleanupWrapper(func() {
for _, c := range cleanups {
c()
}
})()

config, err := test.GrabOAuthServerConfig(kubeClients.CoreV1())
config, err := test.GrabOAuthServerConfig(clients.KubeClient.CoreV1())
require.NoError(t, err)

gitlabIDPConfig := test.GetIDPByName(config, idpName)
Expand Down
Loading