From 49cce7ae449e35fbfbeefea5941165cfd0026c90 Mon Sep 17 00:00:00 2001 From: Yuedong Wu Date: Thu, 21 Nov 2024 17:48:47 +0800 Subject: [PATCH 1/4] add a helper func to dump namespaced events * create DumpEventsInNamespace * tweak CreateTestingNS and DeleteTestingNS logic, move them to BeforeEach() * change the global wait-poll interval to a less aggressive value --- test/e2e/cert_manager_deployment_test.go | 8 +- test/e2e/certificates_test.go | 151 +++++++++++++++++------ test/library/utils.go | 43 ++++--- 3 files changed, 145 insertions(+), 57 deletions(-) diff --git a/test/e2e/cert_manager_deployment_test.go b/test/e2e/cert_manager_deployment_test.go index 52ea65d4e..26d4c27ea 100644 --- a/test/e2e/cert_manager_deployment_test.go +++ b/test/e2e/cert_manager_deployment_test.go @@ -31,7 +31,7 @@ import ( ) const ( - PollInterval = time.Second + PollInterval = 5 * time.Second TestTimeout = 10 * time.Minute ) @@ -44,7 +44,7 @@ func TestSelfSignedCerts(t *testing.T) { ns, err := loader.CreateTestingNS("e2e-self-signed-cert") require.NoError(t, err) - defer loader.DeleteTestingNS(ns.Name) + defer loader.DeleteTestingNS(ns.Name, t.Failed) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "issuer.yaml"), ns.Name) @@ -75,7 +75,7 @@ func TestACMECertsIngress(t *testing.T) { ns, err := loader.CreateTestingNS("e2e-acme-ingress-cert") require.NoError(t, err) - defer loader.DeleteTestingNS(ns.Name) + defer loader.DeleteTestingNS(ns.Name, t.Failed) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "acme", "clusterissuer.yaml"), ns.Name) defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "acme", "clusterissuer.yaml"), ns.Name) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "acme", "deployment.yaml"), ns.Name) @@ -161,7 +161,7 @@ func TestCertRenew(t *testing.T) { ns, err := loader.CreateTestingNS("e2e-cert-renew") require.NoErrorf(t, err, "failed to create namespace: %v", err) - defer loader.DeleteTestingNS(ns.Name) + defer loader.DeleteTestingNS(ns.Name, t.Failed) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "issuer.yaml"), ns.Name) diff --git a/test/e2e/certificates_test.go b/test/e2e/certificates_test.go index ebd2b59ad..07d8d4209 100644 --- a/test/e2e/certificates_test.go +++ b/test/e2e/certificates_test.go @@ -34,6 +34,7 @@ const ( var _ = Describe("ACME Certificate", Ordered, func() { var ctx context.Context + var ns *corev1.Namespace var appsDomain string var baseDomain string @@ -71,16 +72,20 @@ var _ = Describe("ACME Certificate", Ordered, func() { certManagerCAInjectorDeploymentControllerName}, validOperatorStatusConditions) Expect(err).NotTo(HaveOccurred(), "Operator is expected to be available") + + By("creating a test namespace") + namespace, err := loader.CreateTestingNS("e2e-acme-certs") + Expect(err).NotTo(HaveOccurred()) + ns = namespace + + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) }) Context("dns-01 challenge with AWS Route53", Label("Platform:AWS"), func() { It("should obtain a valid LetsEncrypt certificate using explicit credentials", func() { - By("creating a test namespace") - ns, err := loader.CreateTestingNS("e2e-acme-explicit-dns01") - Expect(err).NotTo(HaveOccurred()) - defer loader.DeleteTestingNS(ns.Name) - By("obtaining AWS credentials from kube-system namespace") awsCredsSecret, err := loader.KubeClient.CoreV1().Secrets("kube-system").Get(ctx, "aws-creds", metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -185,16 +190,11 @@ var _ = Describe("ACME Certificate", Ordered, func() { It("should obtain a valid LetsEncrypt certificate using ambient credentials with ClusterIssuer", func() { - By("creating a test namespace") - ns, err := loader.CreateTestingNS("e2e-acme-ambient-dns01") - Expect(err).NotTo(HaveOccurred()) - defer loader.DeleteTestingNS(ns.Name) - By("creating CredentialsRequest object") loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "credentials", "credentialsrequest_aws.yaml"), "") By("waiting for cloud secret to be available") - err = wait.PollImmediate(PollInterval, TestTimeout, func() (bool, error) { + err := wait.PollImmediate(PollInterval, TestTimeout, func() (bool, error) { _, err := loader.KubeClient.CoreV1().Secrets("cert-manager").Get(ctx, "aws-creds", metav1.GetOptions{}) if err != nil { return false, nil @@ -282,16 +282,11 @@ var _ = Describe("ACME Certificate", Ordered, func() { It("should obtain a valid LetsEncrypt certificate using ambient credentials with Issuer", func() { - By("creating a test namespace") - ns, err := loader.CreateTestingNS("e2e-acme-issuer-ambient-dns01-aws") - Expect(err).NotTo(HaveOccurred()) - defer loader.DeleteTestingNS(ns.Name) - By("creating CredentialsRequest object") loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "credentials", "credentialsrequest_aws.yaml"), "") By("waiting for cloud secret to be available") - err = wait.PollImmediate(PollInterval, TestTimeout, func() (bool, error) { + err := wait.PollImmediate(PollInterval, TestTimeout, func() (bool, error) { _, err := loader.KubeClient.CoreV1().Secrets("cert-manager").Get(ctx, "aws-creds", metav1.GetOptions{}) if err != nil { return false, nil @@ -381,12 +376,108 @@ var _ = Describe("ACME Certificate", Ordered, func() { }) Context("dns-01 challenge with Google CloudDNS", Label("Platform:GCP"), func() { - It("should obtain a valid LetsEncrypt certificate using ambient credentials with ClusterIssuer", func() { + It("should obtain a valid LetsEncrypt certificate using explicit credentials with ClusterIssuer", func() { + + By("obtaining GCP credentials from kube-system namespace") + gcpCredsSecret, err := loader.KubeClient.CoreV1().Secrets("kube-system").Get(ctx, "gcp-credentials", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + gcpServiceAccount := gcpCredsSecret.Data["service_account.json"] + + By("copying GCP secret service account to test namespace") + secretName := "gcp-secret" + secretKey := "gcp_service_account_key.json" + gcpSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: ns.Name, + }, + Data: map[string][]byte{ + secretKey: gcpServiceAccount, + }, + } + _, err = loader.KubeClient.CoreV1().Secrets(ns.Name).Create(ctx, gcpSecret, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) - By("Creating a test namespace") - ns, err := loader.CreateTestingNS("e2e-acme-ambient-dns01") + By("getting GCP project ID from Infrastructure object") + infra, err := configClient.Infrastructures().Get(ctx, "cluster", metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) - defer loader.DeleteTestingNS(ns.Name) + + gcpProjectID := infra.Status.PlatformStatus.GCP.ProjectID + Expect(gcpProjectID).NotTo(Equal("")) + + By("creating new certificate Issuer") + issuerName := "letsencrypt-dns01" + issuer := &certmanagerv1.Issuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: issuerName, + Namespace: ns.Name, + }, + Spec: certmanagerv1.IssuerSpec{ + IssuerConfig: certmanagerv1.IssuerConfig{ + ACME: &v1.ACMEIssuer{ + Server: "https://acme-staging-v02.api.letsencrypt.org/directory", + PrivateKey: certmanagermetav1.SecretKeySelector{ + LocalObjectReference: certmanagermetav1.LocalObjectReference{ + Name: "letsencrypt-dns01-issuer", + }, + }, + Solvers: []v1.ACMEChallengeSolver{ + { + DNS01: &v1.ACMEChallengeSolverDNS01{ + CloudDNS: &v1.ACMEIssuerDNS01ProviderCloudDNS{ + Project: string(gcpProjectID), + ServiceAccount: &certmanagermetav1.SecretKeySelector{ + LocalObjectReference: certmanagermetav1.LocalObjectReference{ + Name: secretName, + }, + Key: secretKey, + }, + }, + }, + }, + }, + }, + }, + }, + } + _, err = certmanagerClient.CertmanagerV1().Issuers(ns.Name).Create(ctx, issuer, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + defer certmanagerClient.CertmanagerV1().Issuers(ns.Name).Delete(ctx, issuerName, metav1.DeleteOptions{}) + + By("creating new certificate") + randomString := randomStr(3) + certDomain := randomString + "." + appsDomain + certName := "letsencrypt-cert" + cert := &certmanagerv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: certName, + Namespace: ns.Name, + }, + Spec: certmanagerv1.CertificateSpec{ + IsCA: false, + CommonName: certDomain, + SecretName: certName, + DNSNames: []string{certDomain}, + IssuerRef: certmanagermetav1.ObjectReference{ + Name: issuerName, + Kind: "Issuer", + }, + }, + } + _, err = certmanagerClient.CertmanagerV1().Certificates(ns.Name).Create(ctx, cert, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + defer certmanagerClient.CertmanagerV1().Certificates(ns.Name).Delete(ctx, certName, metav1.DeleteOptions{}) + + By("waiting for certificate to get ready") + err = waitForCertificateReadiness(ctx, certName, ns.Name) + Expect(err).NotTo(HaveOccurred()) + + By("checking for certificate validity from secret contents") + err = verifyCertificate(ctx, certName, ns.Name, certDomain) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should obtain a valid LetsEncrypt certificate using ambient credentials with ClusterIssuer", func() { By("Creating CredentialsRequest object") loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "credentials", "credentialsrequest_gcp.yaml"), "") @@ -394,7 +485,7 @@ var _ = Describe("ACME Certificate", Ordered, func() { By("Waiting for cloud secret to be available") // The name is defined cloud credential by the testdata YAML file. credentialSecret := "gcp-credentials" - err = wait.PollImmediate(PollInterval, TestTimeout, func() (bool, error) { + err := wait.PollImmediate(PollInterval, TestTimeout, func() (bool, error) { _, err := loader.KubeClient.CoreV1().Secrets("cert-manager").Get(ctx, credentialSecret, metav1.GetOptions{}) if err != nil { return false, nil @@ -465,11 +556,6 @@ var _ = Describe("ACME Certificate", Ordered, func() { Skip("skipping as the cluster does not use IBM Cloud CIS") } - By("creating a test namespace") - ns, err := loader.CreateTestingNS("e2e-acme-explicit-dns01-ibmcloud") - Expect(err).NotTo(HaveOccurred()) - defer loader.DeleteTestingNS(ns.Name) - By("creating new certificate ClusterIssuer with IBM Cloud CIS webhook solver") randomString := randomStr(3) clusterIssuerName := "letsencrypt-dns01-explicit-ic" @@ -498,7 +584,7 @@ var _ = Describe("ACME Certificate", Ordered, func() { loader.CreateFromFile(loadFileAndReplaceStr, filepath.Join("testdata", "acme", "certificate_ibmcis.yaml"), ns.Name) By("waiting for certificate to get ready") - err = waitForCertificateReadiness(ctx, certName, ns.Name) + err := waitForCertificateReadiness(ctx, certName, ns.Name) Expect(err).NotTo(HaveOccurred()) By("checking for certificate validity from secret contents") @@ -510,11 +596,6 @@ var _ = Describe("ACME Certificate", Ordered, func() { Context("http-01 challenge using ingress", func() { It("should obtain a valid LetsEncrypt certificate", func() { - By("creating a test namespace") - ns, err := loader.CreateTestingNS("e2e-acme-explicit-dns01") - Expect(err).NotTo(HaveOccurred()) - defer loader.DeleteTestingNS(ns.Name) - By("creating a cluster issuer") loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "acme", "clusterissuer.yaml"), ns.Name) defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "acme", "clusterissuer.yaml"), ns.Name) @@ -567,7 +648,7 @@ var _ = Describe("ACME Certificate", Ordered, func() { }}, }, } - ingress, err = loader.KubeClient.NetworkingV1().Ingresses(ingress.ObjectMeta.Namespace).Create(ctx, ingress, metav1.CreateOptions{}) + ingress, err := loader.KubeClient.NetworkingV1().Ingresses(ingress.ObjectMeta.Namespace).Create(ctx, ingress, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) defer loader.KubeClient.NetworkingV1().Ingresses(ingress.ObjectMeta.Namespace).Delete(ctx, ingress.ObjectMeta.Name, metav1.DeleteOptions{}) @@ -607,7 +688,7 @@ var _ = Describe("Self-signed Certificate", Ordered, func() { ns = namespace DeferCleanup(func() { - loader.DeleteTestingNS(ns.Name) + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) }) }) diff --git a/test/library/utils.go b/test/library/utils.go index 826af82f9..60e3ef7f0 100644 --- a/test/library/utils.go +++ b/test/library/utils.go @@ -6,7 +6,7 @@ package library import ( "context" "fmt" - "testing" + "log" "time" v1 "k8s.io/api/core/v1" @@ -18,7 +18,6 @@ import ( ) func (d DynamicResourceLoader) CreateTestingNS(namespacePrefix string) (*v1.Namespace, error) { - t := testing.T{} namespace := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ GenerateName: fmt.Sprintf("%v-", namespacePrefix), @@ -34,46 +33,54 @@ func (d DynamicResourceLoader) CreateTestingNS(namespacePrefix string) (*v1.Name var err error got, err = d.KubeClient.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) if err != nil { - t.Logf("Error creating namespace: %v", err) + log.Printf("Error creating namespace: %v", err) return false, nil } return true, nil }); err != nil { return nil, err } - return got, nil } -func (d DynamicResourceLoader) DeleteTestingNS(name string) (bool, error) { - t := testing.T{} +func (d DynamicResourceLoader) DeleteTestingNS(name string, shouldDumpEvents func() bool) (bool, error) { ctx := context.Background() + if shouldDumpEvents() { + d.DumpEventsInNamespace(name) + } err := d.KubeClient.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{}) if err != nil { - t.Logf("Namespace: %v not found, err: %v", name, err) + log.Printf("Error deleting namespace %v, err: %v", name, err) } if err := wait.PollImmediate(1*time.Second, 30*time.Second, func() (bool, error) { - // Poll until namespace is deleted - ns, err := d.KubeClient.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{}) - t.Logf("Namespace: %v", ns) - if err != nil { - t.Logf("Error getting namespace: %v", err) - if k8serrors.IsNotFound(err) { - return true, err - } - return false, nil + _, err := d.KubeClient.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{}) + if err != nil && k8serrors.IsNotFound(err) { + return true, nil } return false, nil }); err != nil { - t.Logf("Error getting namespace: %v", err) - return true, err + log.Printf("Timed out after 30s waiting for namespace %v to become deleted", name) + return false, err } return false, nil } +func (d DynamicResourceLoader) DumpEventsInNamespace(name string) { + log.Printf("Dumping events in namespace %s...", name) + events, err := d.KubeClient.CoreV1().Events(name).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + log.Printf("Error listing events in namespace %s: %v", name, err) + return + } + + for _, e := range events.Items { + log.Printf("At %v - event for %v %v: %v %v: %v", e.FirstTimestamp, e.InvolvedObject.Kind, e.InvolvedObject.Name, e.Source, e.Reason, e.Message) + } +} + func GetClusterBaseDomain(ctx context.Context, configClient configv1.ConfigV1Interface) (string, error) { dns, err := configClient.DNSes().Get(ctx, "cluster", metav1.GetOptions{}) if err != nil { From f50eb6356666d88c3b4a43a06c8e9df0207522ae Mon Sep 17 00:00:00 2001 From: Swarup Ghosh Date: Sun, 23 Feb 2025 17:44:14 +0530 Subject: [PATCH 2/4] refactor: Use go text/template in e2e testdata for extensible replacement of field values Signed-off-by: Swarup Ghosh --- test/e2e/certificates_test.go | 56 ++++++++----------- test/e2e/config_template.go | 55 ++++++++++++++++++ test/e2e/testdata/acme/certificate_gcp.yaml | 4 +- .../e2e/testdata/acme/certificate_ibmcis.yaml | 4 +- test/e2e/testdata/acme/clusterissuer_gcp.yaml | 2 +- .../testdata/acme/clusterissuer_ibmcis.yaml | 3 +- 6 files changed, 83 insertions(+), 41 deletions(-) create mode 100644 test/e2e/config_template.go diff --git a/test/e2e/certificates_test.go b/test/e2e/certificates_test.go index 07d8d4209..bc6ad0e60 100644 --- a/test/e2e/certificates_test.go +++ b/test/e2e/certificates_test.go @@ -5,6 +5,7 @@ package e2e import ( "context" + "fmt" "os" "path/filepath" "time" @@ -508,29 +509,22 @@ var _ = Describe("ACME Certificate", Ordered, func() { By("Creating new certificate ClusterIssuer") // The name is defined by the testdata YAML file clusterissuer_gcp.yaml clusterIssuerName := "acme-dns01-clouddns-ambient" - replaceStrMap := map[string]string{ - "PROJECT_ID": gcpProjectId, - } - loadFileAndReplaceStr := func(fileName string) ([]byte, error) { - fileContentsStr, err := replaceStrInFile(replaceStrMap, fileName) - return []byte(fileContentsStr), err - } - loader.CreateFromFile(loadFileAndReplaceStr, filepath.Join("testdata", "acme", "clusterissuer_gcp.yaml"), "") + loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( + IssuerConfig{ + GCPProjectID: gcpProjectId, + }, + ), filepath.Join("testdata", "acme", "clusterissuer_gcp.yaml"), "") defer certmanagerClient.CertmanagerV1().ClusterIssuers().Delete(ctx, clusterIssuerName, metav1.DeleteOptions{}) By("Creating new certificate") randomString := randomStr(3) - replaceStrMap = map[string]string{ - "RANDOM_STR": randomString, - "DNS_NAME": baseDomain, - } - loadFileAndReplaceStr = func(fileName string) ([]byte, error) { - fileContentsStr, err := replaceStrInFile(replaceStrMap, fileName) - return []byte(fileContentsStr), err - } // The name is defined by the testdata YAML file certificate_gcp.yaml certName := "cert-with-acme-dns01-clouddns-ambient" - loader.CreateFromFile(loadFileAndReplaceStr, filepath.Join("testdata", "acme", "certificate_gcp.yaml"), ns.Name) + loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( + CertificateConfig{ + DNSName: fmt.Sprintf("%s.%s", randomString, baseDomain), + }, + ), filepath.Join("testdata", "acme", "certificate_gcp.yaml"), ns.Name) By("Waiting for certificate to get ready") err = waitForCertificateReadiness(ctx, certName, ns.Name) @@ -559,29 +553,23 @@ var _ = Describe("ACME Certificate", Ordered, func() { By("creating new certificate ClusterIssuer with IBM Cloud CIS webhook solver") randomString := randomStr(3) clusterIssuerName := "letsencrypt-dns01-explicit-ic" - replaceStrMap := map[string]string{ - "CIS_CRN": cisCRN, - } - loadFileAndReplaceStr := func(fileName string) ([]byte, error) { - fileContentsStr, err := replaceStrInFile(replaceStrMap, fileName) - return []byte(fileContentsStr), err - } - loader.CreateFromFile(loadFileAndReplaceStr, filepath.Join("testdata", "acme", "clusterissuer_ibmcis.yaml"), "") + loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( + IssuerConfig{ + IBMCloudCISCRN: cisCRN, + }, + ), filepath.Join("testdata", "acme", "clusterissuer_ibmcis.yaml"), "") defer certmanagerClient.CertmanagerV1().ClusterIssuers().Delete(ctx, clusterIssuerName, metav1.DeleteOptions{}) By("creating new certificate") // The name is defined by the testdata YAML file certificate_ibmcis.yaml certDomain := "adwie." + appsDomain // acronym for "ACME dns-01 ibmcloud Webhook Explicit", short naming to pass dns name validation certName := "letsencrypt-cert-ic" - replaceStrMap = map[string]string{ - "RANDOM_STR": randomString, - "DNS_NAME": certDomain, - } - loadFileAndReplaceStr = func(fileName string) ([]byte, error) { - fileContentsStr, err := replaceStrInFile(replaceStrMap, fileName) - return []byte(fileContentsStr), err - } - loader.CreateFromFile(loadFileAndReplaceStr, filepath.Join("testdata", "acme", "certificate_ibmcis.yaml"), ns.Name) + loader.CreateFromFile( + AssetFunc(testassets.ReadFile).WithTemplateValues( + CertificateConfig{ + DNSName: certDomain, + }, + ), filepath.Join("testdata", "acme", "certificate_ibmcis.yaml"), ns.Name) By("waiting for certificate to get ready") err := waitForCertificateReadiness(ctx, certName, ns.Name) diff --git a/test/e2e/config_template.go b/test/e2e/config_template.go new file mode 100644 index 000000000..de4e2cbf5 --- /dev/null +++ b/test/e2e/config_template.go @@ -0,0 +1,55 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "bytes" + "text/template" +) + +// IssuerConfig customizes fields in the issuer spec +type IssuerConfig struct { + GCPProjectID string + IBMCloudCISCRN string +} + +// Certificate customize fields in the cert spec +type CertificateConfig struct { + DNSName string +} + +// replaceWithTemplate puts field values from a template struct +func replaceWithTemplate(sourceFileContents string, templatedValues any) ([]byte, error) { + tmpl, err := template.New("template").Parse(sourceFileContents) + if err != nil { + return nil, err + } + + var doc bytes.Buffer + err = tmpl.Execute(&doc, templatedValues) + if err != nil { + return nil, err + } + + return doc.Bytes(), nil +} + +// AssetFunc wraps the asset load function (used in dynamic resource loader), +// and extends it with a hook to allow template value replacement. +type AssetFunc func(name string) ([]byte, error) + +// WithTemplateValues is a wrapper for using `replaceWithTemplate` with an `AssetFunc`, +// i.e. chains the loading -> modification. +func (sourceFn AssetFunc) WithTemplateValues(templatedValues any) AssetFunc { + x := func(name string) ([]byte, error) { + bytes, err := sourceFn(name) + if err != nil { + return nil, err + } + + fileContentsStr := string(bytes) + return replaceWithTemplate(fileContentsStr, templatedValues) + } + return x +} diff --git a/test/e2e/testdata/acme/certificate_gcp.yaml b/test/e2e/testdata/acme/certificate_gcp.yaml index 267eafe20..e1f731192 100644 --- a/test/e2e/testdata/acme/certificate_gcp.yaml +++ b/test/e2e/testdata/acme/certificate_gcp.yaml @@ -9,5 +9,5 @@ spec: kind: ClusterIssuer name: acme-dns01-clouddns-ambient dnsNames: - - RANDOM_STR.DNS_NAME - - '*.RANDOM_STR.DNS_NAME' + - {{.DNSName}} + - '*.{{.DNSName}}' diff --git a/test/e2e/testdata/acme/certificate_ibmcis.yaml b/test/e2e/testdata/acme/certificate_ibmcis.yaml index 744f9f72e..630ead242 100644 --- a/test/e2e/testdata/acme/certificate_ibmcis.yaml +++ b/test/e2e/testdata/acme/certificate_ibmcis.yaml @@ -4,8 +4,8 @@ metadata: name: letsencrypt-cert-ic spec: dnsNames: - - RANDOM_STR.DNS_NAME - - '*.RANDOM_STR.DNS_NAME' + - {{.DNSName}} + - '*.{{.DNSName}}' issuerRef: name: letsencrypt-dns01-explicit-ic kind: ClusterIssuer diff --git a/test/e2e/testdata/acme/clusterissuer_gcp.yaml b/test/e2e/testdata/acme/clusterissuer_gcp.yaml index f7005436c..78281f8f0 100644 --- a/test/e2e/testdata/acme/clusterissuer_gcp.yaml +++ b/test/e2e/testdata/acme/clusterissuer_gcp.yaml @@ -12,4 +12,4 @@ spec: - dns01: cloudDNS: # The ID of the GCP project - project: PROJECT_ID + project: {{.GCPProjectId}} diff --git a/test/e2e/testdata/acme/clusterissuer_ibmcis.yaml b/test/e2e/testdata/acme/clusterissuer_ibmcis.yaml index 88173f2e3..60f832701 100644 --- a/test/e2e/testdata/acme/clusterissuer_ibmcis.yaml +++ b/test/e2e/testdata/acme/clusterissuer_ibmcis.yaml @@ -17,5 +17,4 @@ spec: name: ibmcis-credentials key: api-token cisCRN: - - "CIS_CRN" - \ No newline at end of file + - "{{.IBMCloudCISCRN}}" From f951da67e3c978d666d99868357d848a0df9170f Mon Sep 17 00:00:00 2001 From: Manish Pillai Date: Mon, 13 Jan 2025 14:12:49 +0530 Subject: [PATCH 3/4] adds e2e test case for istio-csr:#423 - applys IstioCSR resource - deploys grpcurl job - which calls the grpc endpoint of istio-csr - checks the response and validates the certificate --- test/e2e/cert_manager_deployment_test.go | 6 +- test/e2e/certificates_test.go | 4 +- test/e2e/istio_csr_test.go | 287 +++++++++++++++++++ test/e2e/testdata/istio/istio_ca_issuer.yaml | 8 + test/e2e/testdata/istio/istio_csr.yaml | 16 ++ test/e2e/utils_test.go | 131 +++++++++ test/library/utils.go | 93 +++++- 7 files changed, 535 insertions(+), 10 deletions(-) create mode 100644 test/e2e/istio_csr_test.go create mode 100644 test/e2e/testdata/istio/istio_ca_issuer.yaml create mode 100644 test/e2e/testdata/istio/istio_csr.yaml diff --git a/test/e2e/cert_manager_deployment_test.go b/test/e2e/cert_manager_deployment_test.go index 26d4c27ea..06de87257 100644 --- a/test/e2e/cert_manager_deployment_test.go +++ b/test/e2e/cert_manager_deployment_test.go @@ -42,7 +42,7 @@ func TestSelfSignedCerts(t *testing.T) { ctx := context.Background() loader := library.NewDynamicResourceLoader(ctx, t) - ns, err := loader.CreateTestingNS("e2e-self-signed-cert") + ns, err := loader.CreateTestingNS("e2e-self-signed-cert", false) require.NoError(t, err) defer loader.DeleteTestingNS(ns.Name, t.Failed) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) @@ -73,7 +73,7 @@ func TestACMECertsIngress(t *testing.T) { config, err := library.GetConfigForTest(t) require.NoError(t, err) - ns, err := loader.CreateTestingNS("e2e-acme-ingress-cert") + ns, err := loader.CreateTestingNS("e2e-acme-ingress-cert", false) require.NoError(t, err) defer loader.DeleteTestingNS(ns.Name, t.Failed) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "acme", "clusterissuer.yaml"), ns.Name) @@ -159,7 +159,7 @@ func TestCertRenew(t *testing.T) { config, err := library.GetConfigForTest(t) require.NoErrorf(t, err, "failed to fetch host configuration: %v", err) - ns, err := loader.CreateTestingNS("e2e-cert-renew") + ns, err := loader.CreateTestingNS("e2e-cert-renew", false) require.NoErrorf(t, err, "failed to create namespace: %v", err) defer loader.DeleteTestingNS(ns.Name, t.Failed) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) diff --git a/test/e2e/certificates_test.go b/test/e2e/certificates_test.go index bc6ad0e60..8722edc49 100644 --- a/test/e2e/certificates_test.go +++ b/test/e2e/certificates_test.go @@ -75,7 +75,7 @@ var _ = Describe("ACME Certificate", Ordered, func() { Expect(err).NotTo(HaveOccurred(), "Operator is expected to be available") By("creating a test namespace") - namespace, err := loader.CreateTestingNS("e2e-acme-certs") + namespace, err := loader.CreateTestingNS("e2e-acme-certs", false) Expect(err).NotTo(HaveOccurred()) ns = namespace @@ -671,7 +671,7 @@ var _ = Describe("Self-signed Certificate", Ordered, func() { ctx = context.Background() By("creating a test namespace") - namespace, err := loader.CreateTestingNS("e2e-self-signed-certs") + namespace, err := loader.CreateTestingNS("e2e-self-signed-certs", false) Expect(err).NotTo(HaveOccurred()) ns = namespace diff --git a/test/e2e/istio_csr_test.go b/test/e2e/istio_csr_test.go new file mode 100644 index 000000000..0d5e0d441 --- /dev/null +++ b/test/e2e/istio_csr_test.go @@ -0,0 +1,287 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "context" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "fmt" + "io" + "net/url" + "path/filepath" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + + "github.com/openshift/cert-manager-operator/test/library" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// backOffLimit is the max retries for the Job +const backOffLimit int32 = 10 + +// istioCSRProtoURL links to proto for istio-csr API spec +const istioCSRProtoURL = "https://raw.githubusercontent.com/istio/api/v1.24.1/security/v1alpha1/ca.proto" + +type LogEntry struct { + CertChain []string `json:"certChain"` +} + +var _ = Describe("Istio-CSR", Ordered, Label("TechPreview", "Feature:IstioCSR"), func() { + ctx := context.TODO() + var clientset *kubernetes.Clientset + var dynamicClient *dynamic.DynamicClient + + BeforeAll(func() { + var err error + clientset, err = kubernetes.NewForConfig(cfg) + Expect(err).Should(BeNil()) + + dynamicClient, err = dynamic.NewForConfig(cfg) + Expect(err).Should(BeNil()) + }) + + var ns *corev1.Namespace + + BeforeEach(func() { + By("waiting for operator status to become available") + err := verifyOperatorStatusCondition(certmanageroperatorclient, []string{ + certManagerControllerDeploymentControllerName, + certManagerWebhookDeploymentControllerName, + certManagerCAInjectorDeploymentControllerName, + }, validOperatorStatusConditions) + Expect(err).NotTo(HaveOccurred(), "Operator is expected to be available") + + By("creating a test namespace") + namespace, err := loader.CreateTestingNS("istio-system", true) + Expect(err).NotTo(HaveOccurred()) + ns = namespace + + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + }) + + Context("grpc call istio.v1.auth.IstioCertificateService/CreateCertificate to istio-csr agent", func() { + It("should return cert-chain as response", func() { + serviceAccountName := "cert-manager-istio-csr" + grpcAppName := "grpcurl" + + By("creating cluster issuer") + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) + defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) + + By("issuing TLS certificate") + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "certificate.yaml"), ns.Name) + defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "certificate.yaml"), ns.Name) + + By("fetching proto file from api") + protoContent, err := library.FetchFileFromURL(istioCSRProtoURL) + Expect(err).Should(BeNil()) + Expect(protoContent).NotTo(BeEmpty()) + + By("creating proto config map") + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proto-cm", + Namespace: ns.Name, + }, + Data: map[string]string{ + "ca.proto": protoContent, + }, + } + _, err = clientset.CoreV1().ConfigMaps(ns.Name).Create(ctx, configMap, metav1.CreateOptions{}) + Expect(err).Should(BeNil()) + defer clientset.CoreV1().ConfigMaps(ns.Name).Delete(ctx, configMap.Name, metav1.DeleteOptions{}) + + By("creating istio-ca issuer") + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_ca_issuer.yaml"), ns.Name) + defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_ca_issuer.yaml"), ns.Name) + + By("creating istiocsr.operator.openshift.io resource") + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_csr.yaml"), ns.Name) + defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_csr.yaml"), ns.Name) + + By("poll till cert-manager-istio-csr is available") + err = pollTillDeploymentAvailable(ctx, clientset, ns.Name, "cert-manager-istio-csr") + Expect(err).Should(BeNil()) + + istioCSRGRPCEndpoint, err := pollTillIstioCSRAvailable(ctx, dynamicClient, ns.Name, "default") + Expect(err).Should(BeNil()) + + By("poll till the service account is available") + err = pollTillServiceAccountAvailable(ctx, clientset, ns.Name, serviceAccountName) + Expect(err).Should(BeNil()) + + By("generate csr request") + + csrTemplate := &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"My Organization"}, + OrganizationalUnit: []string{"IT Department"}, + Country: []string{"US"}, + Locality: []string{"Los Angeles"}, + Province: []string{"California"}, + }, + URIs: []*url.URL{ + {Scheme: "spiffe", Host: "cluster.local", Path: "/ns/istio-system/sa/cert-manager-istio-csr"}, + }, + SignatureAlgorithm: x509.SHA256WithRSA, + } + + csr, err := library.GenerateCSR(csrTemplate) + Expect(err).Should(BeNil()) + + By("creating an grpcurl job") + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "grpcurl-job", + }, + Spec: batchv1.JobSpec{ + Completions: ptr.To(int32(1)), + BackoffLimit: ptr.To(backOffLimit), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: grpcAppName, + Labels: map[string]string{ + "app": grpcAppName, + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccountName, + AutomountServiceAccountToken: ptr.To(false), + RestartPolicy: corev1.RestartPolicyOnFailure, + Containers: []corev1.Container{ + { + Name: grpcAppName, + Image: "registry.redhat.io/rhel9/go-toolset", + Command: []string{ + "/bin/sh", + "-c", + }, + Env: []corev1.EnvVar{ + { + Name: "GOCACHE", + Value: "/tmp/go-cache", + }, + { + Name: "GOPATH", + Value: "/tmp/go", + }, + }, + Args: []string{ + "go install github.com/fullstorydev/grpcurl/cmd/grpcurl@v1.9.2 >/dev/null 2>&1 && " + + "TOKEN=$(cat /var/run/secrets/istio-ca/token) && " + + "/tmp/go/bin/grpcurl " + + "-import-path /proto " + + "-proto /proto/ca.proto " + + "-H \"Authorization: Bearer $TOKEN\" " + + fmt.Sprintf("-d '{\"csr\": \"%s\", \"validity_duration\": 3600}' ", csr) + + "-cacert /etc/root-secret/ca.crt " + + "-key /etc/root-secret/tls.key " + + "-cert /etc/root-secret/tls.crt " + + fmt.Sprintf("%s istio.v1.auth.IstioCertificateService/CreateCertificate", istioCSRGRPCEndpoint), + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "root-secret", MountPath: "/etc/root-secret"}, + {Name: "proto", MountPath: "/proto"}, + {Name: "sa-token", MountPath: "/var/run/secrets/istio-ca"}, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "sa-token", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: ptr.To(int32(420)), + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: "istio-ca", + ExpirationSeconds: ptr.To(int64(3600)), + Path: "token", + }, + }, + }, + }, + }, + }, + { + Name: "root-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "istiod-tls", + }, + }, + }, + { + Name: "proto", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "proto-cm", + }, + }, + }, + }, + }, + }, + }, + }, + } + _, err = clientset.BatchV1().Jobs(ns.Name).Create(context.TODO(), job, metav1.CreateOptions{}) + Expect(err).Should(BeNil()) + defer clientset.BatchV1().Jobs(ns.Name).Delete(ctx, job.Name, metav1.DeleteOptions{}) + + By("waiting for the job to be completed") + err = pollTillJobCompleted(ctx, clientset, ns.Name, "grpcurl-job") + Expect(err).Should(BeNil()) + + By("fetching logs of the grpcurl job") + pods, err := clientset.CoreV1().Pods(ns.Name).List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", grpcAppName), + }) + Expect(err).Should(BeNil()) + + By("fetching succeeded pod name") + var succeededPodName string + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodSucceeded { + succeededPodName = pod.Name + } + } + Expect(succeededPodName).ShouldNot(BeEmpty()) + + req := clientset.CoreV1().Pods(ns.Name).GetLogs(succeededPodName, &corev1.PodLogOptions{}) + logs, err := req.Stream(context.TODO()) + Expect(err).Should(BeNil()) + + defer logs.Close() + + logData, err := io.ReadAll(logs) + Expect(err).Should(BeNil()) + + var entry LogEntry + err = json.Unmarshal(logData, &entry) + Expect(err).Should(BeNil()) + Expect(entry.CertChain).ShouldNot(BeEmpty()) + + By("validating each certificate") + for _, certPEM := range entry.CertChain { + err = library.ValidateCertificate(certPEM, "my-selfsigned-ca") + Expect(err).Should(BeNil()) + } + + }) + }) +}) diff --git a/test/e2e/testdata/istio/istio_ca_issuer.yaml b/test/e2e/testdata/istio/istio_ca_issuer.yaml new file mode 100644 index 000000000..b7e074e18 --- /dev/null +++ b/test/e2e/testdata/istio/istio_ca_issuer.yaml @@ -0,0 +1,8 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: istio-ca + namespace: istio-system +spec: + ca: + secretName: root-secret \ No newline at end of file diff --git a/test/e2e/testdata/istio/istio_csr.yaml b/test/e2e/testdata/istio/istio_csr.yaml new file mode 100644 index 000000000..d06776539 --- /dev/null +++ b/test/e2e/testdata/istio/istio_csr.yaml @@ -0,0 +1,16 @@ +apiVersion: operator.openshift.io/v1alpha1 +kind: IstioCSR +metadata: + name: default + namespace: istio-system +spec: + istioCSRConfig: + certManager: + issuerRef: + name: istio-ca + kind: Issuer + group: cert-manager.io + istiodTLSConfig: + trustDomain: cluster.local + istio: + namespace: istio-system \ No newline at end of file diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 21d834826..cde2d42cf 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "fmt" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "math/rand" "regexp" "strings" @@ -25,6 +26,8 @@ import ( certmanoperatorclient "github.com/openshift/cert-manager-operator/pkg/operator/clientset/versioned" "github.com/openshift/cert-manager-operator/test/library" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -36,6 +39,7 @@ import ( "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/retry" ) @@ -564,3 +568,130 @@ func waitForIngressReadiness(ctx context.Context, client kubernetes.Interface, i return false, nil }) } + +// pollTillJobCompleted poll the job object and returns non-nil error +// once the job is completed, otherwise should return a time-out error +func pollTillJobCompleted(ctx context.Context, clientset *kubernetes.Clientset, namespace, jobName string) error { + err := wait.PollUntilContextTimeout(ctx, PollInterval, TestTimeout, true, func(ctx context.Context) (bool, error) { + job, err := clientset.BatchV1().Jobs(namespace).Get(ctx, jobName, metav1.GetOptions{}) + + if err != nil { + return false, err + } + + for _, cond := range job.Status.Conditions { + if cond.Type == batchv1.JobComplete { + if cond.Status == corev1.ConditionTrue { + return true, nil + } else { + return false, nil + } + } + } + + return false, nil + }) + return err +} + +// pollTillServiceAccountAvailable poll the service account object and returns non-nil error +// once the service account is available, otherwise should return a time-out error +func pollTillServiceAccountAvailable(ctx context.Context, clientset *kubernetes.Clientset, namespace, serviceAccountName string) error { + err := wait.PollUntilContextTimeout(ctx, PollInterval, TestTimeout, true, func(ctx context.Context) (bool, error) { + _, err := clientset.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + return true, nil + }) + + return err +} + +// pollTillIstioCSRAvailable poll the istioCSR object and returns non-nil error and istio-grpc-endpoint +// once the istiocsr is available, otherwise should return a time-out error +func pollTillIstioCSRAvailable(ctx context.Context, dynamicClient *dynamic.DynamicClient, namespace, istioCsrName string) (string, error) { + var istioCSRGRPCEndpoint string + err := wait.PollUntilContextTimeout(ctx, PollInterval, TestTimeout, true, func(ctx context.Context) (bool, error) { + gvr := schema.GroupVersionResource{ + Group: "operator.openshift.io", + Version: "v1alpha1", + Resource: "istiocsrs", + } + + customResource, err := dynamicClient.Resource(gvr).Namespace(namespace).Get(ctx, istioCsrName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + + status, found, err := unstructured.NestedMap(customResource.Object, "status") + if err != nil { + return false, nil + } + + if !found { + return false, nil + } + + conditions, found, err := unstructured.NestedSlice(customResource.Object, "status", "conditions") + if err != nil { + return false, nil + } + + if !found { + return false, nil + } + + for _, condition := range conditions { + condMap, ok := condition.(map[string]interface{}) + if !ok { + continue + } + + condType, _ := condMap["type"].(string) + condStatus, _ := condMap["status"].(string) + + if condType != "Ready" { + continue + } + + if condStatus == string(metav1.ConditionTrue) { + break + } else { + return false, nil + } + + } + + if !library.IsEmptyString(status["istioCSRGRPCEndpoint"]) && !library.IsEmptyString(status["clusterRoleBinding"]) && !library.IsEmptyString(status["istioCSRImage"]) && !library.IsEmptyString(status["serviceAccount"]) { + istioCSRGRPCEndpoint = status["istioCSRGRPCEndpoint"].(string) + return true, nil + } + return false, nil + }) + + return istioCSRGRPCEndpoint, err +} + +func pollTillDeploymentAvailable(ctx context.Context, clientSet *kubernetes.Clientset, namespace, deploymentName string) error { + err := wait.PollUntilContextTimeout(ctx, PollInterval, TestTimeout, true, func(ctx context.Context) (bool, error) { + deployment, err := clientSet.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + + for _, cond := range deployment.Status.Conditions { + if cond.Type == appsv1.DeploymentAvailable { + return cond.Status == corev1.ConditionTrue, nil + } + } + + return false, nil + }) + + return err +} diff --git a/test/library/utils.go b/test/library/utils.go index 60e3ef7f0..989f140ed 100644 --- a/test/library/utils.go +++ b/test/library/utils.go @@ -5,11 +5,18 @@ package library import ( "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" "fmt" + "io" "log" + "net/http" + "strings" "time" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -17,10 +24,9 @@ import ( configv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" ) -func (d DynamicResourceLoader) CreateTestingNS(namespacePrefix string) (*v1.Namespace, error) { - namespace := &v1.Namespace{ +func (d DynamicResourceLoader) CreateTestingNS(namespacePrefix string, noSuffix bool) (*corev1.Namespace, error) { + namespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("%v-", namespacePrefix), Labels: map[string]string{ "e2e-test": "true", "operator": "openshift-cert-manager-operator", @@ -28,7 +34,13 @@ func (d DynamicResourceLoader) CreateTestingNS(namespacePrefix string) (*v1.Name }, } - var got *v1.Namespace + if noSuffix { + namespace.ObjectMeta.Name = namespacePrefix + } else { + namespace.ObjectMeta.GenerateName = fmt.Sprintf("%v-", namespacePrefix) + } + + var got *corev1.Namespace if err := wait.PollImmediate(1*time.Second, 30*time.Second, func() (bool, error) { var err error got, err = d.KubeClient.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}) @@ -88,3 +100,74 @@ func GetClusterBaseDomain(ctx context.Context, configClient configv1.ConfigV1Int } return dns.Spec.BaseDomain, nil } + +func ValidateCertificate(certPem string, expectedCommonName string) error { + block, _ := pem.Decode([]byte(certPem)) + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return err + } + + if cert.Issuer.CommonName != expectedCommonName { + return fmt.Errorf("expected common name %v, got %v", expectedCommonName, cert.Subject.CommonName) + } + + now := time.Now() + if now.Before(cert.NotBefore) || now.After(cert.NotAfter) { + return fmt.Errorf("certificate is not valid yet") + } + + return nil +} + +func FetchFileFromURL(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", fmt.Errorf("failed to GET the URL: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("received non-200 response: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %w", err) + } + + return string(body), nil +} + +func IsEmptyString(key interface{}) bool { + if key == nil { + return true + } + + if key.(string) == "" { + return true + } + + return false +} + +func GenerateCSR(csrTemplate *x509.CertificateRequest) (string, error) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return "", fmt.Errorf("failed to generate private key: %w", err) + } + + csrBytes, err := x509.CreateCertificateRequest(rand.Reader, csrTemplate, privateKey) + if err != nil { + return "", fmt.Errorf("failed to create CSR: %w", err) + } + + csrPEM := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE REQUEST", + Bytes: csrBytes, + }) + + escapedCSR := strings.ReplaceAll(string(csrPEM), "\n", "\\n") + + return escapedCSR, nil +} From c64c3acdc53b2c967e887410670f4dd43a64dd43 Mon Sep 17 00:00:00 2001 From: Manish Pillai Date: Tue, 4 Mar 2025 10:53:38 +0530 Subject: [PATCH 4/4] replaces grpcurl job with loader file --- test/e2e/config_template.go | 8 ++ test/e2e/istio_csr_test.go | 113 ++--------------------- test/e2e/testdata/istio/grpcurl_job.yaml | 62 +++++++++++++ test/e2e/utils_test.go | 73 ++++++--------- 4 files changed, 106 insertions(+), 150 deletions(-) create mode 100644 test/e2e/testdata/istio/grpcurl_job.yaml diff --git a/test/e2e/config_template.go b/test/e2e/config_template.go index de4e2cbf5..18778f606 100644 --- a/test/e2e/config_template.go +++ b/test/e2e/config_template.go @@ -6,6 +6,8 @@ package e2e import ( "bytes" "text/template" + + "github.com/openshift/cert-manager-operator/api/operator/v1alpha1" ) // IssuerConfig customizes fields in the issuer spec @@ -19,6 +21,12 @@ type CertificateConfig struct { DNSName string } +// IstioCSRConfig customizes the fields in a job spec +type IstioCSRGRPCurlJobConfig struct { + CertificateSigningRequest string + IstioCSRStatus v1alpha1.IstioCSRStatus +} + // replaceWithTemplate puts field values from a template struct func replaceWithTemplate(sourceFileContents string, templatedValues any) ([]byte, error) { tmpl, err := template.New("template").Parse(sourceFileContents) diff --git a/test/e2e/istio_csr_test.go b/test/e2e/istio_csr_test.go index 0d5e0d441..aa592d7f3 100644 --- a/test/e2e/istio_csr_test.go +++ b/test/e2e/istio_csr_test.go @@ -13,12 +13,10 @@ import ( "net/url" "path/filepath" - batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" - "k8s.io/utils/ptr" "github.com/openshift/cert-manager-operator/test/library" @@ -74,7 +72,7 @@ var _ = Describe("Istio-CSR", Ordered, Label("TechPreview", "Feature:IstioCSR"), Context("grpc call istio.v1.auth.IstioCertificateService/CreateCertificate to istio-csr agent", func() { It("should return cert-chain as response", func() { serviceAccountName := "cert-manager-istio-csr" - grpcAppName := "grpcurl" + grpcAppName := "grpcurl-istio-csr" By("creating cluster issuer") loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) @@ -115,7 +113,7 @@ var _ = Describe("Istio-CSR", Ordered, Label("TechPreview", "Feature:IstioCSR"), err = pollTillDeploymentAvailable(ctx, clientset, ns.Name, "cert-manager-istio-csr") Expect(err).Should(BeNil()) - istioCSRGRPCEndpoint, err := pollTillIstioCSRAvailable(ctx, dynamicClient, ns.Name, "default") + istioCSRStatus, err := pollTillIstioCSRAvailable(ctx, dynamicClient, ns.Name, "default") Expect(err).Should(BeNil()) By("poll till the service account is available") @@ -142,109 +140,16 @@ var _ = Describe("Istio-CSR", Ordered, Label("TechPreview", "Feature:IstioCSR"), Expect(err).Should(BeNil()) By("creating an grpcurl job") - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: "grpcurl-job", - }, - Spec: batchv1.JobSpec{ - Completions: ptr.To(int32(1)), - BackoffLimit: ptr.To(backOffLimit), - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: grpcAppName, - Labels: map[string]string{ - "app": grpcAppName, - }, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: serviceAccountName, - AutomountServiceAccountToken: ptr.To(false), - RestartPolicy: corev1.RestartPolicyOnFailure, - Containers: []corev1.Container{ - { - Name: grpcAppName, - Image: "registry.redhat.io/rhel9/go-toolset", - Command: []string{ - "/bin/sh", - "-c", - }, - Env: []corev1.EnvVar{ - { - Name: "GOCACHE", - Value: "/tmp/go-cache", - }, - { - Name: "GOPATH", - Value: "/tmp/go", - }, - }, - Args: []string{ - "go install github.com/fullstorydev/grpcurl/cmd/grpcurl@v1.9.2 >/dev/null 2>&1 && " + - "TOKEN=$(cat /var/run/secrets/istio-ca/token) && " + - "/tmp/go/bin/grpcurl " + - "-import-path /proto " + - "-proto /proto/ca.proto " + - "-H \"Authorization: Bearer $TOKEN\" " + - fmt.Sprintf("-d '{\"csr\": \"%s\", \"validity_duration\": 3600}' ", csr) + - "-cacert /etc/root-secret/ca.crt " + - "-key /etc/root-secret/tls.key " + - "-cert /etc/root-secret/tls.crt " + - fmt.Sprintf("%s istio.v1.auth.IstioCertificateService/CreateCertificate", istioCSRGRPCEndpoint), - }, - VolumeMounts: []corev1.VolumeMount{ - {Name: "root-secret", MountPath: "/etc/root-secret"}, - {Name: "proto", MountPath: "/proto"}, - {Name: "sa-token", MountPath: "/var/run/secrets/istio-ca"}, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "sa-token", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - DefaultMode: ptr.To(int32(420)), - Sources: []corev1.VolumeProjection{ - { - ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ - Audience: "istio-ca", - ExpirationSeconds: ptr.To(int64(3600)), - Path: "token", - }, - }, - }, - }, - }, - }, - { - Name: "root-secret", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "istiod-tls", - }, - }, - }, - { - Name: "proto", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "proto-cm", - }, - }, - }, - }, - }, - }, - }, + loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( + IstioCSRGRPCurlJobConfig{ + CertificateSigningRequest: csr, + IstioCSRStatus: istioCSRStatus, }, - } - _, err = clientset.BatchV1().Jobs(ns.Name).Create(context.TODO(), job, metav1.CreateOptions{}) - Expect(err).Should(BeNil()) - defer clientset.BatchV1().Jobs(ns.Name).Delete(ctx, job.Name, metav1.DeleteOptions{}) + ), filepath.Join("testdata", "istio", "grpcurl_job.yaml"), ns.Name) + defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "grpcurl_job.yaml"), ns.Name) By("waiting for the job to be completed") - err = pollTillJobCompleted(ctx, clientset, ns.Name, "grpcurl-job") + err = pollTillJobCompleted(ctx, clientset, ns.Name, grpcAppName) Expect(err).Should(BeNil()) By("fetching logs of the grpcurl job") diff --git a/test/e2e/testdata/istio/grpcurl_job.yaml b/test/e2e/testdata/istio/grpcurl_job.yaml new file mode 100644 index 000000000..ef5802a15 --- /dev/null +++ b/test/e2e/testdata/istio/grpcurl_job.yaml @@ -0,0 +1,62 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: grpcurl-istio-csr +spec: + backoffLimit: 10 + completions: 1 + template: + metadata: + labels: + app: grpcurl-istio-csr + name: grpcurl-istio-csr + spec: + automountServiceAccountToken: false + containers: + - args: + - | + go install github.com/fullstorydev/grpcurl/cmd/grpcurl@v1.9.2 >/dev/null 2>&1 && \ + TOKEN=$(cat /var/run/secrets/istio-ca/token) && \ + /tmp/go/bin/grpcurl \ + -import-path /proto \ + -proto /proto/ca.proto \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"csr": "{{.CertificateSigningRequest}}", "validity_duration": 3600}' \ + -cacert /etc/root-secret/ca.crt \ + -key /etc/root-secret/tls.key \ + -cert /etc/root-secret/tls.crt \ + {{.IstioCSRStatus.IstioCSRGRPCEndpoint}} istio.v1.auth.IstioCertificateService/CreateCertificate + command: + - /bin/sh + - -c + env: + - name: GOCACHE + value: /tmp/go-cache + - name: GOPATH + value: /tmp/go + image: registry.redhat.io/rhel9/go-toolset + name: grpcurl + volumeMounts: + - mountPath: /etc/root-secret + name: root-secret + - mountPath: /proto + name: proto + - mountPath: /var/run/secrets/istio-ca + name: sa-token + restartPolicy: OnFailure + serviceAccountName: '{{.IstioCSRStatus.ServiceAccount}}' + volumes: + - name: sa-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: istio-ca + expirationSeconds: 3600 + path: token + - name: root-secret + secret: + secretName: istiod-tls + - configMap: + name: proto-cm + name: proto diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index cde2d42cf..a745378da 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -8,7 +8,6 @@ import ( "context" "encoding/json" "fmt" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "math/rand" "regexp" "strings" @@ -32,8 +31,10 @@ import ( networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/errors" @@ -62,7 +63,7 @@ func verifyOperatorStatusCondition(client *certmanoperatorclient.Clientset, cont go func(index int) { defer wg.Done() err := wait.PollImmediate(time.Second*1, time.Minute*5, func() (done bool, err error) { - operator, err := client.OperatorV1alpha1().CertManagers().Get(context.TODO(), "cluster", v1.GetOptions{}) + operator, err := client.OperatorV1alpha1().CertManagers().Get(context.TODO(), "cluster", metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return false, nil @@ -98,7 +99,7 @@ func resetCertManagerState(ctx context.Context, client *certmanoperatorclient.Cl err := retry.RetryOnConflict(retry.DefaultRetry, func() error { var operatorState *v1alpha1.CertManager err := wait.PollImmediate(PollInterval, TestTimeout, func() (bool, error) { - operator, err := client.OperatorV1alpha1().CertManagers().Get(ctx, "cluster", v1.GetOptions{}) + operator, err := client.OperatorV1alpha1().CertManagers().Get(ctx, "cluster", metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return false, nil @@ -122,7 +123,7 @@ func resetCertManagerState(ctx context.Context, client *certmanoperatorclient.Cl ManagementState: opv1.Managed, } - _, err = client.OperatorV1alpha1().CertManagers().Update(context.TODO(), updatedOperator, v1.UpdateOptions{}) + _, err = client.OperatorV1alpha1().CertManagers().Update(context.TODO(), updatedOperator, metav1.UpdateOptions{}) return err }) @@ -146,7 +147,7 @@ func resetCertManagerState(ctx context.Context, client *certmanoperatorclient.Cl } subscriptionClient := loader.DynamicClient.Resource(subscriptionSchema).Namespace("cert-manager-operator") - _, err = subscriptionClient.Patch(ctx, subName, types.MergePatchType, payload, v1.PatchOptions{}) + _, err = subscriptionClient.Patch(ctx, subName, types.MergePatchType, payload, metav1.PatchOptions{}) return err } @@ -154,7 +155,7 @@ func resetCertManagerState(ctx context.Context, client *certmanoperatorclient.Cl // a conflict error is encountered. func addOverrideArgs(client *certmanoperatorclient.Clientset, deploymentName string, args []string) error { return retry.RetryOnConflict(retry.DefaultRetry, func() error { - operator, err := client.OperatorV1alpha1().CertManagers().Get(context.TODO(), "cluster", v1.GetOptions{}) + operator, err := client.OperatorV1alpha1().CertManagers().Get(context.TODO(), "cluster", metav1.GetOptions{}) if err != nil { return err } @@ -178,7 +179,7 @@ func addOverrideArgs(client *certmanoperatorclient.Clientset, deploymentName str return fmt.Errorf("unsupported deployment name: %s", deploymentName) } - _, err = client.OperatorV1alpha1().CertManagers().Update(context.TODO(), updatedOperator, v1.UpdateOptions{}) + _, err = client.OperatorV1alpha1().CertManagers().Update(context.TODO(), updatedOperator, metav1.UpdateOptions{}) return err }) } @@ -189,7 +190,7 @@ func addOverrideArgs(client *certmanoperatorclient.Clientset, deploymentName str func verifyDeploymentArgs(k8sclient *kubernetes.Clientset, deploymentName string, args []string, added bool) error { return wait.PollImmediate(time.Second*1, time.Minute*5, func() (done bool, err error) { - controllerDeployment, err := k8sclient.AppsV1().Deployments(operandNamespace).Get(context.TODO(), deploymentName, v1.GetOptions{}) + controllerDeployment, err := k8sclient.AppsV1().Deployments(operandNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return false, nil @@ -221,7 +222,7 @@ func verifyDeploymentArgs(k8sclient *kubernetes.Clientset, deploymentName string // is retried if a conflict error is encountered. func addOverrideResources(client *certmanoperatorclient.Clientset, deploymentName string, res v1alpha1.CertManagerResourceRequirements) error { return retry.RetryOnConflict(retry.DefaultRetry, func() error { - operator, err := client.OperatorV1alpha1().CertManagers().Get(context.TODO(), "cluster", v1.GetOptions{}) + operator, err := client.OperatorV1alpha1().CertManagers().Get(context.TODO(), "cluster", metav1.GetOptions{}) if err != nil { return err } @@ -245,7 +246,7 @@ func addOverrideResources(client *certmanoperatorclient.Clientset, deploymentNam return fmt.Errorf("unsupported deployment name: %s", deploymentName) } - _, err = client.OperatorV1alpha1().CertManagers().Update(context.TODO(), updatedOperator, v1.UpdateOptions{}) + _, err = client.OperatorV1alpha1().CertManagers().Update(context.TODO(), updatedOperator, metav1.UpdateOptions{}) return err }) } @@ -256,7 +257,7 @@ func addOverrideResources(client *certmanoperatorclient.Clientset, deploymentNam func verifyDeploymentResources(k8sclient *kubernetes.Clientset, deploymentName string, res v1alpha1.CertManagerResourceRequirements, added bool) error { return wait.PollImmediate(time.Second*10, time.Minute*5, func() (done bool, err error) { - controllerDeployment, err := k8sclient.AppsV1().Deployments(operandNamespace).Get(context.TODO(), deploymentName, v1.GetOptions{}) + controllerDeployment, err := k8sclient.AppsV1().Deployments(operandNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return false, nil @@ -292,7 +293,7 @@ func verifyDeploymentResources(k8sclient *kubernetes.Clientset, deploymentName s // is retried if a conflict error is encountered. func addOverrideScheduling(client *certmanoperatorclient.Clientset, deploymentName string, res v1alpha1.CertManagerScheduling) error { return retry.RetryOnConflict(retry.DefaultRetry, func() error { - operator, err := client.OperatorV1alpha1().CertManagers().Get(context.TODO(), "cluster", v1.GetOptions{}) + operator, err := client.OperatorV1alpha1().CertManagers().Get(context.TODO(), "cluster", metav1.GetOptions{}) if err != nil { return err } @@ -316,7 +317,7 @@ func addOverrideScheduling(client *certmanoperatorclient.Clientset, deploymentNa return fmt.Errorf("unsupported deployment name: %s", deploymentName) } - _, err = client.OperatorV1alpha1().CertManagers().Update(context.TODO(), updatedOperator, v1.UpdateOptions{}) + _, err = client.OperatorV1alpha1().CertManagers().Update(context.TODO(), updatedOperator, metav1.UpdateOptions{}) return err }) } @@ -327,7 +328,7 @@ func addOverrideScheduling(client *certmanoperatorclient.Clientset, deploymentNa func verifyDeploymentScheduling(k8sclient *kubernetes.Clientset, deploymentName string, res v1alpha1.CertManagerScheduling, added bool) error { return wait.PollUntilContextTimeout(context.Background(), time.Second*10, time.Minute*5, true, func(context.Context) (done bool, err error) { - controllerDeployment, err := k8sclient.AppsV1().Deployments(operandNamespace).Get(context.TODO(), deploymentName, v1.GetOptions{}) + controllerDeployment, err := k8sclient.AppsV1().Deployments(operandNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return false, nil @@ -384,7 +385,7 @@ func verifyDeploymentScheduling(k8sclient *kubernetes.Clientset, deploymentName func getCertManagerOperatorSubscription(ctx context.Context, loader library.DynamicResourceLoader) (string, error) { subscriptionClient := loader.DynamicClient.Resource(subscriptionSchema).Namespace("cert-manager-operator") - subs, err := subscriptionClient.List(ctx, v1.ListOptions{}) + subs, err := subscriptionClient.List(ctx, metav1.ListOptions{}) if err != nil { return "", err } @@ -426,7 +427,7 @@ func patchSubscriptionWithCloudCredential(ctx context.Context, loader library.Dy } subscriptionClient := loader.DynamicClient.Resource(subscriptionSchema).Namespace("cert-manager-operator") - _, err = subscriptionClient.Patch(ctx, subName, types.MergePatchType, payload, v1.PatchOptions{}) + _, err = subscriptionClient.Patch(ctx, subName, types.MergePatchType, payload, metav1.PatchOptions{}) return err } @@ -612,10 +613,10 @@ func pollTillServiceAccountAvailable(ctx context.Context, clientset *kubernetes. return err } -// pollTillIstioCSRAvailable poll the istioCSR object and returns non-nil error and istio-grpc-endpoint +// pollTillIstioCSRAvailable poll the istioCSR object and returns non-nil error and istioCSRStatus // once the istiocsr is available, otherwise should return a time-out error -func pollTillIstioCSRAvailable(ctx context.Context, dynamicClient *dynamic.DynamicClient, namespace, istioCsrName string) (string, error) { - var istioCSRGRPCEndpoint string +func pollTillIstioCSRAvailable(ctx context.Context, dynamicClient *dynamic.DynamicClient, namespace, istioCsrName string) (v1alpha1.IstioCSRStatus, error) { + var istioCSRStatus v1alpha1.IstioCSRStatus err := wait.PollUntilContextTimeout(ctx, PollInterval, TestTimeout, true, func(ctx context.Context) (bool, error) { gvr := schema.GroupVersionResource{ Group: "operator.openshift.io", @@ -637,44 +638,24 @@ func pollTillIstioCSRAvailable(ctx context.Context, dynamicClient *dynamic.Dynam return false, nil } - conditions, found, err := unstructured.NestedSlice(customResource.Object, "status", "conditions") + err = runtime.DefaultUnstructuredConverter.FromUnstructured(status, &istioCSRStatus) if err != nil { return false, nil } - if !found { - return false, nil - } - - for _, condition := range conditions { - condMap, ok := condition.(map[string]interface{}) - if !ok { - continue - } - - condType, _ := condMap["type"].(string) - condStatus, _ := condMap["status"].(string) - - if condType != "Ready" { - continue - } - - if condStatus == string(metav1.ConditionTrue) { - break - } else { - return false, nil - } + readyCondition := meta.FindStatusCondition(istioCSRStatus.Conditions, v1alpha1.Ready) + if readyCondition == nil || readyCondition.Status != metav1.ConditionTrue { + return false, nil } - if !library.IsEmptyString(status["istioCSRGRPCEndpoint"]) && !library.IsEmptyString(status["clusterRoleBinding"]) && !library.IsEmptyString(status["istioCSRImage"]) && !library.IsEmptyString(status["serviceAccount"]) { - istioCSRGRPCEndpoint = status["istioCSRGRPCEndpoint"].(string) + if !library.IsEmptyString(istioCSRStatus.IstioCSRGRPCEndpoint) && !library.IsEmptyString(istioCSRStatus.ClusterRoleBinding) && !library.IsEmptyString(istioCSRStatus.IstioCSRImage) && !library.IsEmptyString(istioCSRStatus.ServiceAccount) { return true, nil } return false, nil }) - return istioCSRGRPCEndpoint, err + return istioCSRStatus, err } func pollTillDeploymentAvailable(ctx context.Context, clientSet *kubernetes.Clientset, namespace, deploymentName string) error {