From efa7e23248bb2926bded74f7c6f40430ff785619 Mon Sep 17 00:00:00 2001 From: Manish Pillai Date: Mon, 13 Jan 2025 14:12:49 +0530 Subject: [PATCH] adds e2e test case for istio-csr:#423 - applys IstioCSR resource - deploys grpcurl job - which calls the grpc endpoint of istio-csr - checks the response and validates the certificate --- test/e2e/cert_manager_deployment_test.go | 6 +- test/e2e/certificates_test.go | 4 +- test/e2e/istio_csr_test.go | 287 +++++++++++++++++++ test/e2e/testdata/istio/istio_ca_issuer.yaml | 8 + test/e2e/testdata/istio/istio_csr.yaml | 16 ++ test/e2e/utils_test.go | 131 +++++++++ test/library/utils.go | 87 +++++- 7 files changed, 532 insertions(+), 7 deletions(-) create mode 100644 test/e2e/istio_csr_test.go create mode 100644 test/e2e/testdata/istio/istio_ca_issuer.yaml create mode 100644 test/e2e/testdata/istio/istio_csr.yaml diff --git a/test/e2e/cert_manager_deployment_test.go b/test/e2e/cert_manager_deployment_test.go index 26d4c27ea..06de87257 100644 --- a/test/e2e/cert_manager_deployment_test.go +++ b/test/e2e/cert_manager_deployment_test.go @@ -42,7 +42,7 @@ func TestSelfSignedCerts(t *testing.T) { ctx := context.Background() loader := library.NewDynamicResourceLoader(ctx, t) - ns, err := loader.CreateTestingNS("e2e-self-signed-cert") + ns, err := loader.CreateTestingNS("e2e-self-signed-cert", false) require.NoError(t, err) defer loader.DeleteTestingNS(ns.Name, t.Failed) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) @@ -73,7 +73,7 @@ func TestACMECertsIngress(t *testing.T) { config, err := library.GetConfigForTest(t) require.NoError(t, err) - ns, err := loader.CreateTestingNS("e2e-acme-ingress-cert") + ns, err := loader.CreateTestingNS("e2e-acme-ingress-cert", false) require.NoError(t, err) defer loader.DeleteTestingNS(ns.Name, t.Failed) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "acme", "clusterissuer.yaml"), ns.Name) @@ -159,7 +159,7 @@ func TestCertRenew(t *testing.T) { config, err := library.GetConfigForTest(t) require.NoErrorf(t, err, "failed to fetch host configuration: %v", err) - ns, err := loader.CreateTestingNS("e2e-cert-renew") + ns, err := loader.CreateTestingNS("e2e-cert-renew", false) require.NoErrorf(t, err, "failed to create namespace: %v", err) defer loader.DeleteTestingNS(ns.Name, t.Failed) loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) diff --git a/test/e2e/certificates_test.go b/test/e2e/certificates_test.go index 3d72a9391..82405f2f2 100644 --- a/test/e2e/certificates_test.go +++ b/test/e2e/certificates_test.go @@ -74,7 +74,7 @@ var _ = Describe("ACME Certificate", Ordered, func() { Expect(err).NotTo(HaveOccurred(), "Operator is expected to be available") By("creating a test namespace") - namespace, err := loader.CreateTestingNS("e2e-acme-certs") + namespace, err := loader.CreateTestingNS("e2e-acme-certs", false) Expect(err).NotTo(HaveOccurred()) ns = namespace @@ -683,7 +683,7 @@ var _ = Describe("Self-signed Certificate", Ordered, func() { ctx = context.Background() By("creating a test namespace") - namespace, err := loader.CreateTestingNS("e2e-self-signed-certs") + namespace, err := loader.CreateTestingNS("e2e-self-signed-certs", false) Expect(err).NotTo(HaveOccurred()) ns = namespace diff --git a/test/e2e/istio_csr_test.go b/test/e2e/istio_csr_test.go new file mode 100644 index 000000000..0d5e0d441 --- /dev/null +++ b/test/e2e/istio_csr_test.go @@ -0,0 +1,287 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "context" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "fmt" + "io" + "net/url" + "path/filepath" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + + "github.com/openshift/cert-manager-operator/test/library" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// backOffLimit is the max retries for the Job +const backOffLimit int32 = 10 + +// istioCSRProtoURL links to proto for istio-csr API spec +const istioCSRProtoURL = "https://raw.githubusercontent.com/istio/api/v1.24.1/security/v1alpha1/ca.proto" + +type LogEntry struct { + CertChain []string `json:"certChain"` +} + +var _ = Describe("Istio-CSR", Ordered, Label("TechPreview", "Feature:IstioCSR"), func() { + ctx := context.TODO() + var clientset *kubernetes.Clientset + var dynamicClient *dynamic.DynamicClient + + BeforeAll(func() { + var err error + clientset, err = kubernetes.NewForConfig(cfg) + Expect(err).Should(BeNil()) + + dynamicClient, err = dynamic.NewForConfig(cfg) + Expect(err).Should(BeNil()) + }) + + var ns *corev1.Namespace + + BeforeEach(func() { + By("waiting for operator status to become available") + err := verifyOperatorStatusCondition(certmanageroperatorclient, []string{ + certManagerControllerDeploymentControllerName, + certManagerWebhookDeploymentControllerName, + certManagerCAInjectorDeploymentControllerName, + }, validOperatorStatusConditions) + Expect(err).NotTo(HaveOccurred(), "Operator is expected to be available") + + By("creating a test namespace") + namespace, err := loader.CreateTestingNS("istio-system", true) + Expect(err).NotTo(HaveOccurred()) + ns = namespace + + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + }) + + Context("grpc call istio.v1.auth.IstioCertificateService/CreateCertificate to istio-csr agent", func() { + It("should return cert-chain as response", func() { + serviceAccountName := "cert-manager-istio-csr" + grpcAppName := "grpcurl" + + By("creating cluster issuer") + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) + defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "cluster_issuer.yaml"), ns.Name) + + By("issuing TLS certificate") + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "certificate.yaml"), ns.Name) + defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "certificate.yaml"), ns.Name) + + By("fetching proto file from api") + protoContent, err := library.FetchFileFromURL(istioCSRProtoURL) + Expect(err).Should(BeNil()) + Expect(protoContent).NotTo(BeEmpty()) + + By("creating proto config map") + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proto-cm", + Namespace: ns.Name, + }, + Data: map[string]string{ + "ca.proto": protoContent, + }, + } + _, err = clientset.CoreV1().ConfigMaps(ns.Name).Create(ctx, configMap, metav1.CreateOptions{}) + Expect(err).Should(BeNil()) + defer clientset.CoreV1().ConfigMaps(ns.Name).Delete(ctx, configMap.Name, metav1.DeleteOptions{}) + + By("creating istio-ca issuer") + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_ca_issuer.yaml"), ns.Name) + defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_ca_issuer.yaml"), ns.Name) + + By("creating istiocsr.operator.openshift.io resource") + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_csr.yaml"), ns.Name) + defer loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_csr.yaml"), ns.Name) + + By("poll till cert-manager-istio-csr is available") + err = pollTillDeploymentAvailable(ctx, clientset, ns.Name, "cert-manager-istio-csr") + Expect(err).Should(BeNil()) + + istioCSRGRPCEndpoint, err := pollTillIstioCSRAvailable(ctx, dynamicClient, ns.Name, "default") + Expect(err).Should(BeNil()) + + By("poll till the service account is available") + err = pollTillServiceAccountAvailable(ctx, clientset, ns.Name, serviceAccountName) + Expect(err).Should(BeNil()) + + By("generate csr request") + + csrTemplate := &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"My Organization"}, + OrganizationalUnit: []string{"IT Department"}, + Country: []string{"US"}, + Locality: []string{"Los Angeles"}, + Province: []string{"California"}, + }, + URIs: []*url.URL{ + {Scheme: "spiffe", Host: "cluster.local", Path: "/ns/istio-system/sa/cert-manager-istio-csr"}, + }, + SignatureAlgorithm: x509.SHA256WithRSA, + } + + csr, err := library.GenerateCSR(csrTemplate) + Expect(err).Should(BeNil()) + + By("creating an grpcurl job") + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: "grpcurl-job", + }, + Spec: batchv1.JobSpec{ + Completions: ptr.To(int32(1)), + BackoffLimit: ptr.To(backOffLimit), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: grpcAppName, + Labels: map[string]string{ + "app": grpcAppName, + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccountName, + AutomountServiceAccountToken: ptr.To(false), + RestartPolicy: corev1.RestartPolicyOnFailure, + Containers: []corev1.Container{ + { + Name: grpcAppName, + Image: "registry.redhat.io/rhel9/go-toolset", + Command: []string{ + "/bin/sh", + "-c", + }, + Env: []corev1.EnvVar{ + { + Name: "GOCACHE", + Value: "/tmp/go-cache", + }, + { + Name: "GOPATH", + Value: "/tmp/go", + }, + }, + Args: []string{ + "go install github.com/fullstorydev/grpcurl/cmd/grpcurl@v1.9.2 >/dev/null 2>&1 && " + + "TOKEN=$(cat /var/run/secrets/istio-ca/token) && " + + "/tmp/go/bin/grpcurl " + + "-import-path /proto " + + "-proto /proto/ca.proto " + + "-H \"Authorization: Bearer $TOKEN\" " + + fmt.Sprintf("-d '{\"csr\": \"%s\", \"validity_duration\": 3600}' ", csr) + + "-cacert /etc/root-secret/ca.crt " + + "-key /etc/root-secret/tls.key " + + "-cert /etc/root-secret/tls.crt " + + fmt.Sprintf("%s istio.v1.auth.IstioCertificateService/CreateCertificate", istioCSRGRPCEndpoint), + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "root-secret", MountPath: "/etc/root-secret"}, + {Name: "proto", MountPath: "/proto"}, + {Name: "sa-token", MountPath: "/var/run/secrets/istio-ca"}, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "sa-token", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: ptr.To(int32(420)), + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: "istio-ca", + ExpirationSeconds: ptr.To(int64(3600)), + Path: "token", + }, + }, + }, + }, + }, + }, + { + Name: "root-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "istiod-tls", + }, + }, + }, + { + Name: "proto", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "proto-cm", + }, + }, + }, + }, + }, + }, + }, + }, + } + _, err = clientset.BatchV1().Jobs(ns.Name).Create(context.TODO(), job, metav1.CreateOptions{}) + Expect(err).Should(BeNil()) + defer clientset.BatchV1().Jobs(ns.Name).Delete(ctx, job.Name, metav1.DeleteOptions{}) + + By("waiting for the job to be completed") + err = pollTillJobCompleted(ctx, clientset, ns.Name, "grpcurl-job") + Expect(err).Should(BeNil()) + + By("fetching logs of the grpcurl job") + pods, err := clientset.CoreV1().Pods(ns.Name).List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", grpcAppName), + }) + Expect(err).Should(BeNil()) + + By("fetching succeeded pod name") + var succeededPodName string + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodSucceeded { + succeededPodName = pod.Name + } + } + Expect(succeededPodName).ShouldNot(BeEmpty()) + + req := clientset.CoreV1().Pods(ns.Name).GetLogs(succeededPodName, &corev1.PodLogOptions{}) + logs, err := req.Stream(context.TODO()) + Expect(err).Should(BeNil()) + + defer logs.Close() + + logData, err := io.ReadAll(logs) + Expect(err).Should(BeNil()) + + var entry LogEntry + err = json.Unmarshal(logData, &entry) + Expect(err).Should(BeNil()) + Expect(entry.CertChain).ShouldNot(BeEmpty()) + + By("validating each certificate") + for _, certPEM := range entry.CertChain { + err = library.ValidateCertificate(certPEM, "my-selfsigned-ca") + Expect(err).Should(BeNil()) + } + + }) + }) +}) diff --git a/test/e2e/testdata/istio/istio_ca_issuer.yaml b/test/e2e/testdata/istio/istio_ca_issuer.yaml new file mode 100644 index 000000000..b7e074e18 --- /dev/null +++ b/test/e2e/testdata/istio/istio_ca_issuer.yaml @@ -0,0 +1,8 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: istio-ca + namespace: istio-system +spec: + ca: + secretName: root-secret \ No newline at end of file diff --git a/test/e2e/testdata/istio/istio_csr.yaml b/test/e2e/testdata/istio/istio_csr.yaml new file mode 100644 index 000000000..d06776539 --- /dev/null +++ b/test/e2e/testdata/istio/istio_csr.yaml @@ -0,0 +1,16 @@ +apiVersion: operator.openshift.io/v1alpha1 +kind: IstioCSR +metadata: + name: default + namespace: istio-system +spec: + istioCSRConfig: + certManager: + issuerRef: + name: istio-ca + kind: Issuer + group: cert-manager.io + istiodTLSConfig: + trustDomain: cluster.local + istio: + namespace: istio-system \ No newline at end of file diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 6c2aed21d..b8a6c5061 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "fmt" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "math/rand" "regexp" "strings" @@ -25,6 +26,8 @@ import ( certmanoperatorclient "github.com/openshift/cert-manager-operator/pkg/operator/clientset/versioned" "github.com/openshift/cert-manager-operator/test/library" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -35,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/retry" ) @@ -563,3 +567,130 @@ func waitForIngressReadiness(ctx context.Context, client kubernetes.Interface, i return false, nil }) } + +// pollTillJobCompleted poll the job object and returns non-nil error +// once the job is completed, otherwise should return a time-out error +func pollTillJobCompleted(ctx context.Context, clientset *kubernetes.Clientset, namespace, jobName string) error { + err := wait.PollUntilContextTimeout(ctx, PollInterval, TestTimeout, true, func(ctx context.Context) (bool, error) { + job, err := clientset.BatchV1().Jobs(namespace).Get(ctx, jobName, metav1.GetOptions{}) + + if err != nil { + return false, err + } + + for _, cond := range job.Status.Conditions { + if cond.Type == batchv1.JobComplete { + if cond.Status == corev1.ConditionTrue { + return true, nil + } else { + return false, nil + } + } + } + + return false, nil + }) + return err +} + +// pollTillServiceAccountAvailable poll the service account object and returns non-nil error +// once the service account is available, otherwise should return a time-out error +func pollTillServiceAccountAvailable(ctx context.Context, clientset *kubernetes.Clientset, namespace, serviceAccountName string) error { + err := wait.PollUntilContextTimeout(ctx, PollInterval, TestTimeout, true, func(ctx context.Context) (bool, error) { + _, err := clientset.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + return true, nil + }) + + return err +} + +// pollTillIstioCSRAvailable poll the istioCSR object and returns non-nil error and istio-grpc-endpoint +// once the istiocsr is available, otherwise should return a time-out error +func pollTillIstioCSRAvailable(ctx context.Context, dynamicClient *dynamic.DynamicClient, namespace, istioCsrName string) (string, error) { + var istioCSRGRPCEndpoint string + err := wait.PollUntilContextTimeout(ctx, PollInterval, TestTimeout, true, func(ctx context.Context) (bool, error) { + gvr := schema.GroupVersionResource{ + Group: "operator.openshift.io", + Version: "v1alpha1", + Resource: "istiocsrs", + } + + customResource, err := dynamicClient.Resource(gvr).Namespace(namespace).Get(ctx, istioCsrName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + + status, found, err := unstructured.NestedMap(customResource.Object, "status") + if err != nil { + return false, nil + } + + if !found { + return false, nil + } + + conditions, found, err := unstructured.NestedSlice(customResource.Object, "status", "conditions") + if err != nil { + return false, nil + } + + if !found { + return false, nil + } + + for _, condition := range conditions { + condMap, ok := condition.(map[string]interface{}) + if !ok { + continue + } + + condType, _ := condMap["type"].(string) + condStatus, _ := condMap["status"].(string) + + if condType != "Ready" { + continue + } + + if condStatus == string(metav1.ConditionTrue) { + break + } else { + return false, nil + } + + } + + if !library.IsEmptyString(status["istioCSRGRPCEndpoint"]) && !library.IsEmptyString(status["clusterRoleBinding"]) && !library.IsEmptyString(status["istioCSRImage"]) && !library.IsEmptyString(status["serviceAccount"]) { + istioCSRGRPCEndpoint = status["istioCSRGRPCEndpoint"].(string) + return true, nil + } + return false, nil + }) + + return istioCSRGRPCEndpoint, err +} + +func pollTillDeploymentAvailable(ctx context.Context, clientSet *kubernetes.Clientset, namespace, deploymentName string) error { + err := wait.PollUntilContextTimeout(ctx, PollInterval, TestTimeout, true, func(ctx context.Context) (bool, error) { + deployment, err := clientSet.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + + for _, cond := range deployment.Status.Conditions { + if cond.Type == appsv1.DeploymentAvailable { + return cond.Status == corev1.ConditionTrue, nil + } + } + + return false, nil + }) + + return err +} diff --git a/test/library/utils.go b/test/library/utils.go index 483533989..989f140ed 100644 --- a/test/library/utils.go +++ b/test/library/utils.go @@ -5,8 +5,15 @@ package library import ( "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" "fmt" + "io" "log" + "net/http" + "strings" "time" corev1 "k8s.io/api/core/v1" @@ -17,10 +24,9 @@ import ( configv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" ) -func (d DynamicResourceLoader) CreateTestingNS(namespacePrefix string) (*corev1.Namespace, error) { +func (d DynamicResourceLoader) CreateTestingNS(namespacePrefix string, noSuffix bool) (*corev1.Namespace, error) { namespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("%v-", namespacePrefix), Labels: map[string]string{ "e2e-test": "true", "operator": "openshift-cert-manager-operator", @@ -28,6 +34,12 @@ func (d DynamicResourceLoader) CreateTestingNS(namespacePrefix string) (*corev1. }, } + if noSuffix { + namespace.ObjectMeta.Name = namespacePrefix + } else { + namespace.ObjectMeta.GenerateName = fmt.Sprintf("%v-", namespacePrefix) + } + var got *corev1.Namespace if err := wait.PollImmediate(1*time.Second, 30*time.Second, func() (bool, error) { var err error @@ -88,3 +100,74 @@ func GetClusterBaseDomain(ctx context.Context, configClient configv1.ConfigV1Int } return dns.Spec.BaseDomain, nil } + +func ValidateCertificate(certPem string, expectedCommonName string) error { + block, _ := pem.Decode([]byte(certPem)) + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return err + } + + if cert.Issuer.CommonName != expectedCommonName { + return fmt.Errorf("expected common name %v, got %v", expectedCommonName, cert.Subject.CommonName) + } + + now := time.Now() + if now.Before(cert.NotBefore) || now.After(cert.NotAfter) { + return fmt.Errorf("certificate is not valid yet") + } + + return nil +} + +func FetchFileFromURL(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", fmt.Errorf("failed to GET the URL: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("received non-200 response: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %w", err) + } + + return string(body), nil +} + +func IsEmptyString(key interface{}) bool { + if key == nil { + return true + } + + if key.(string) == "" { + return true + } + + return false +} + +func GenerateCSR(csrTemplate *x509.CertificateRequest) (string, error) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return "", fmt.Errorf("failed to generate private key: %w", err) + } + + csrBytes, err := x509.CreateCertificateRequest(rand.Reader, csrTemplate, privateKey) + if err != nil { + return "", fmt.Errorf("failed to create CSR: %w", err) + } + + csrPEM := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE REQUEST", + Bytes: csrBytes, + }) + + escapedCSR := strings.ReplaceAll(string(csrPEM), "\n", "\\n") + + return escapedCSR, nil +}