From d6e37127f7500a236cc4070368e1593529d6bebc Mon Sep 17 00:00:00 2001 From: Wei Weng Date: Mon, 1 Dec 2025 12:54:03 -0800 Subject: [PATCH 01/16] chore: remove 2 v1alpha1 metrics (#352) remove v1alpha1 metrics Signed-off-by: Wei Weng Co-authored-by: Wei Weng --- pkg/metrics/hub/metrics.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pkg/metrics/hub/metrics.go b/pkg/metrics/hub/metrics.go index c61f85ff0..353cf99ea 100644 --- a/pkg/metrics/hub/metrics.go +++ b/pkg/metrics/hub/metrics.go @@ -23,16 +23,6 @@ import ( ) var ( - // These 2 metrics are used in v1alpha1 controller, should be soon deprecated. - PlacementApplyFailedCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "placement_apply_failed_counter", - Help: "Number of failed to apply cluster resource placement", - }, []string{"name"}) - PlacementApplySucceedCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "placement_apply_succeed_counter", - Help: "Number of successfully applied cluster resource placement", - }, []string{"name"}) - // FleetPlacementStatusLastTimeStampSeconds is a prometheus metric which keeps track of the last placement status. FleetPlacementStatusLastTimeStampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "fleet_workload_placement_status_last_timestamp_seconds", @@ -81,8 +71,6 @@ var ( func init() { metrics.Registry.MustRegister( - PlacementApplyFailedCount, - PlacementApplySucceedCount, FleetPlacementStatusLastTimeStampSeconds, FleetEvictionStatus, FleetUpdateRunStatusLastTimestampSeconds, From 5832681e3a2f21d1b3dff118c4224ce61802ec32 Mon Sep 17 00:00:00 2001 From: Wei Weng Date: Mon, 1 Dec 2025 14:25:04 -0800 Subject: [PATCH 02/16] feat: make hub statefulset work by stripping some properties from generated PVCs (#347) * make statefulset work in hub Signed-off-by: Wei Weng * fix test Signed-off-by: Wei Weng * remove unintended change Signed-off-by: Wei Weng * remove more annotations Signed-off-by: Wei Weng * track pvc availability Signed-off-by: Wei Weng * use cmp.diff to compare status Signed-off-by: Wei Weng * do not propagate PVCs Signed-off-by: Wei Weng * remove PVC annotation logic Signed-off-by: Wei Weng --------- Signed-off-by: Wei Weng Co-authored-by: Wei Weng --- pkg/utils/common.go | 3 + pkg/utils/common_test.go | 13 ++++ test/e2e/enveloped_object_placement_test.go | 2 +- .../e2e/placement_selecting_resources_test.go | 2 +- .../resource_placement_hub_workload_test.go | 75 ++++++++++++++++++- test/e2e/resource_placement_rollout_test.go | 2 +- ...urce_placement_selecting_resources_test.go | 2 +- ...tatefulset.yaml => statefulset-basic.yaml} | 0 ....yaml => statefulset-invalid-storage.yaml} | 0 .../resources/statefulset-with-storage.yaml | 28 +++++++ test/e2e/rollout_test.go | 2 +- test/e2e/utils_test.go | 29 +++++-- 12 files changed, 146 insertions(+), 12 deletions(-) rename test/e2e/resources/{test-statefulset.yaml => statefulset-basic.yaml} (100%) rename test/e2e/resources/{statefulset-with-volume.yaml => statefulset-invalid-storage.yaml} (100%) create mode 100644 test/e2e/resources/statefulset-with-storage.yaml diff --git a/pkg/utils/common.go b/pkg/utils/common.go index cf1899182..63225001b 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -541,6 +541,9 @@ func ShouldPropagateObj(informerManager informer.Manager, uObj *unstructured.Uns if secret.Type == corev1.SecretTypeServiceAccountToken { return false, nil } + case corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim"): + // Skip PersistentVolumeClaims to avoid conflicts with the PVCs created by statefulset controller + return false, nil case corev1.SchemeGroupVersion.WithKind("Endpoints"): // we assume that all endpoints with the same name of a service is created by the service controller if _, err := informerManager.Lister(ServiceGVR).ByNamespace(uObj.GetNamespace()).Get(uObj.GetName()); err != nil { diff --git a/pkg/utils/common_test.go b/pkg/utils/common_test.go index 35716885f..e6f9433d3 100644 --- a/pkg/utils/common_test.go +++ b/pkg/utils/common_test.go @@ -1317,6 +1317,19 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { ownerReferences: nil, want: true, }, + { + name: "PersistentVolumeClaim should NOT propagate", + obj: map[string]interface{}{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": map[string]interface{}{ + "name": "test-pvc", + "namespace": "default", + }, + }, + ownerReferences: nil, + want: false, + }, } for _, tt := range tests { diff --git a/test/e2e/enveloped_object_placement_test.go b/test/e2e/enveloped_object_placement_test.go index b1923b281..b52451958 100644 --- a/test/e2e/enveloped_object_placement_test.go +++ b/test/e2e/enveloped_object_placement_test.go @@ -226,7 +226,7 @@ var _ = Describe("placing wrapped resources using a CRP", func() { // read the test resources. readDeploymentTestManifest(&testDeployment) readDaemonSetTestManifest(&testDaemonSet) - readStatefulSetTestManifest(&testStatefulSet, true) + readStatefulSetTestManifest(&testStatefulSet, StatefulSetInvalidStorage) readEnvelopeResourceTestManifest(&testResourceEnvelope) }) diff --git a/test/e2e/placement_selecting_resources_test.go b/test/e2e/placement_selecting_resources_test.go index 601173394..e060e6920 100644 --- a/test/e2e/placement_selecting_resources_test.go +++ b/test/e2e/placement_selecting_resources_test.go @@ -1392,11 +1392,11 @@ var _ = Describe("creating CRP and checking selected resources order", Ordered, It("should update CRP status with the correct order of the selected resources", func() { // Define the expected resources in order + // Note: PVCs are not propagated, so they should not appear in selected resources expectedResources := []placementv1beta1.ResourceIdentifier{ {Kind: "Namespace", Name: nsName, Version: "v1"}, {Kind: "Secret", Name: secret.Name, Namespace: nsName, Version: "v1"}, {Kind: "ConfigMap", Name: configMap.Name, Namespace: nsName, Version: "v1"}, - {Kind: "PersistentVolumeClaim", Name: pvc.Name, Namespace: nsName, Version: "v1"}, {Group: "rbac.authorization.k8s.io", Kind: "Role", Name: role.Name, Namespace: nsName, Version: "v1"}, } diff --git a/test/e2e/resource_placement_hub_workload_test.go b/test/e2e/resource_placement_hub_workload_test.go index c248319d0..97634f9a8 100644 --- a/test/e2e/resource_placement_hub_workload_test.go +++ b/test/e2e/resource_placement_hub_workload_test.go @@ -19,6 +19,7 @@ package e2e import ( "fmt" + "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" @@ -37,12 +38,14 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res var testDeployment appsv1.Deployment var testDaemonSet appsv1.DaemonSet var testJob batchv1.Job + var testStatefulSet appsv1.StatefulSet BeforeAll(func() { // Read the test manifests readDeploymentTestManifest(&testDeployment) readDaemonSetTestManifest(&testDaemonSet) readJobTestManifest(&testJob) + readStatefulSetTestManifest(&testStatefulSet, StatefulSetWithStorage) workNamespace := appNamespace() // Create namespace and workloads @@ -51,9 +54,11 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res testDeployment.Namespace = workNamespace.Name testDaemonSet.Namespace = workNamespace.Name testJob.Namespace = workNamespace.Name + testStatefulSet.Namespace = workNamespace.Name Expect(hubClient.Create(ctx, &testDeployment)).To(Succeed(), "Failed to create test deployment %s", testDeployment.Name) Expect(hubClient.Create(ctx, &testDaemonSet)).To(Succeed(), "Failed to create test daemonset %s", testDaemonSet.Name) Expect(hubClient.Create(ctx, &testJob)).To(Succeed(), "Failed to create test job %s", testJob.Name) + Expect(hubClient.Create(ctx, &testStatefulSet)).To(Succeed(), "Failed to create test statefulset %s", testStatefulSet.Name) // Create the CRP that selects the namespace By("creating CRP that selects the namespace") @@ -105,9 +110,16 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res Name: testJob.Name, Namespace: workNamespace.Name, }, + { + Group: "apps", + Version: "v1", + Kind: "StatefulSet", + Name: testStatefulSet.Name, + Namespace: workNamespace.Name, + }, } // Use customizedPlacementStatusUpdatedActual with resourceIsTrackable=false - // because Jobs don't have availability tracking like Deployments/DaemonSets do + // because Jobs don't have availability tracking like Deployments/DaemonSets/StatefulSets do crpKey := types.NamespacedName{Name: crpName} crpStatusUpdatedActual := customizedPlacementStatusUpdatedActual(crpKey, wantSelectedResources, allMemberClusterNames, nil, "0", false) Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") @@ -170,6 +182,13 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res "Hub job should complete successfully") }) + It("should verify hub statefulset is ready", func() { + By("checking hub statefulset status") + statefulSetReadyActual := waitForStatefulSetToBeReady(hubClient, &testStatefulSet) + Eventually(statefulSetReadyActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), + "Hub statefulset should be ready before placement") + }) + It("should place the deployment on all member clusters", func() { By("verifying deployment is placed and ready on all member clusters") for idx := range allMemberClusters { @@ -206,6 +225,24 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res } }) + It("should place the statefulset on all member clusters", func() { + By("verifying statefulset is placed and ready on all member clusters") + for idx := range allMemberClusters { + memberCluster := allMemberClusters[idx] + statefulsetPlacedActual := waitForStatefulSetPlacementToReady(memberCluster, &testStatefulSet) + Eventually(statefulsetPlacedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place statefulset on member cluster %s", memberCluster.ClusterName) + } + }) + + It("should verify statefulset replicas are ready on all clusters", func() { + By("checking statefulset status on each cluster") + for _, cluster := range allMemberClusters { + statefulSetReadyActual := waitForStatefulSetToBeReady(cluster.KubeClient, &testStatefulSet) + Eventually(statefulSetReadyActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), + "StatefulSet should be ready on cluster %s", cluster.ClusterName) + } + }) + It("should verify deployment replicas are ready on all clusters", func() { By("checking deployment status on each cluster") for _, cluster := range allMemberClusters { @@ -232,6 +269,42 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res }) }) +func waitForStatefulSetToBeReady(kubeClient client.Client, testStatefulSet *appsv1.StatefulSet) func() error { + return func() error { + var statefulSet appsv1.StatefulSet + if err := kubeClient.Get(ctx, types.NamespacedName{ + Name: testStatefulSet.Name, + Namespace: testStatefulSet.Namespace, + }, &statefulSet); err != nil { + return err + } + + // Verify statefulset is ready + requiredReplicas := int32(1) + if statefulSet.Spec.Replicas != nil { + requiredReplicas = *statefulSet.Spec.Replicas + } + + wantStatus := appsv1.StatefulSetStatus{ + ObservedGeneration: statefulSet.Generation, + CurrentReplicas: requiredReplicas, + UpdatedReplicas: requiredReplicas, + } + + gotStatus := appsv1.StatefulSetStatus{ + ObservedGeneration: statefulSet.Status.ObservedGeneration, + CurrentReplicas: statefulSet.Status.CurrentReplicas, + UpdatedReplicas: statefulSet.Status.UpdatedReplicas, + } + + if diff := cmp.Diff(wantStatus, gotStatus); diff != "" { + return fmt.Errorf("statefulset not ready (-want +got):\n%s", diff) + } + + return nil + } +} + func waitForJobToComplete(kubeClient client.Client, testJob *batchv1.Job) func() error { return func() error { var job batchv1.Job diff --git a/test/e2e/resource_placement_rollout_test.go b/test/e2e/resource_placement_rollout_test.go index 6548a2a15..d780e2d12 100644 --- a/test/e2e/resource_placement_rollout_test.go +++ b/test/e2e/resource_placement_rollout_test.go @@ -69,7 +69,7 @@ var _ = Describe("placing namespaced scoped resources using a RP with rollout", testDaemonSet = appv1.DaemonSet{} readDaemonSetTestManifest(&testDaemonSet) testStatefulSet = appv1.StatefulSet{} - readStatefulSetTestManifest(&testStatefulSet, false) + readStatefulSetTestManifest(&testStatefulSet, StatefulSetBasic) testService = corev1.Service{} readServiceTestManifest(&testService) testJob = batchv1.Job{} diff --git a/test/e2e/resource_placement_selecting_resources_test.go b/test/e2e/resource_placement_selecting_resources_test.go index 069dd3269..8083da339 100644 --- a/test/e2e/resource_placement_selecting_resources_test.go +++ b/test/e2e/resource_placement_selecting_resources_test.go @@ -1013,10 +1013,10 @@ var _ = Describe("testing RP selecting resources", Label("resourceplacement"), f It("should update RP status with the correct order of the selected resources", func() { // Define the expected resources in order. + // Note: PVCs are not propagated, so they should not appear in selected resources expectedResources := []placementv1beta1.ResourceIdentifier{ {Kind: "Secret", Name: secret.Name, Namespace: nsName, Version: "v1"}, {Kind: "ConfigMap", Name: configMap.Name, Namespace: nsName, Version: "v1"}, - {Kind: "PersistentVolumeClaim", Name: pvc.Name, Namespace: nsName, Version: "v1"}, {Group: "rbac.authorization.k8s.io", Kind: "Role", Name: role.Name, Namespace: nsName, Version: "v1"}, } diff --git a/test/e2e/resources/test-statefulset.yaml b/test/e2e/resources/statefulset-basic.yaml similarity index 100% rename from test/e2e/resources/test-statefulset.yaml rename to test/e2e/resources/statefulset-basic.yaml diff --git a/test/e2e/resources/statefulset-with-volume.yaml b/test/e2e/resources/statefulset-invalid-storage.yaml similarity index 100% rename from test/e2e/resources/statefulset-with-volume.yaml rename to test/e2e/resources/statefulset-invalid-storage.yaml diff --git a/test/e2e/resources/statefulset-with-storage.yaml b/test/e2e/resources/statefulset-with-storage.yaml new file mode 100644 index 000000000..a9df178e5 --- /dev/null +++ b/test/e2e/resources/statefulset-with-storage.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-ss +spec: + selector: + matchLabels: + app: test-ss + serviceName: "test-ss-svc" + replicas: 2 + template: + metadata: + labels: + app: test-ss + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: pause + image: k8s.gcr.io/pause:3.8 + volumeClaimTemplates: + - metadata: + name: test-ss-pvc + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "standard" + resources: + requests: + storage: 100Mi diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index 1ff978366..5abfb684f 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -342,7 +342,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { BeforeAll(func() { // Create the test resources. - readStatefulSetTestManifest(&testStatefulSet, false) + readStatefulSetTestManifest(&testStatefulSet, StatefulSetBasic) readEnvelopeResourceTestManifest(&testStatefulSetEnvelope) wantSelectedResources = []placementv1beta1.ResourceIdentifier{ { diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 36df25cf1..b847ceb91 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -57,6 +57,18 @@ import ( "github.com/kubefleet-dev/kubefleet/test/e2e/framework" ) +// StatefulSetVariant represents different StatefulSet configurations for testing +type StatefulSetVariant int + +const ( + // StatefulSetBasic is a StatefulSet without any persistent volume claims + StatefulSetBasic StatefulSetVariant = iota + // StatefulSetInvalidStorage is a StatefulSet with a non-existent storage class + StatefulSetInvalidStorage + // StatefulSetWithStorage is a StatefulSet with a valid standard storage class + StatefulSetWithStorage +) + var ( croTestAnnotationKey = "cro-test-annotation" croTestAnnotationValue = "cro-test-annotation-val" @@ -1537,13 +1549,18 @@ func readDaemonSetTestManifest(testDaemonSet *appsv1.DaemonSet) { Expect(err).Should(Succeed()) } -func readStatefulSetTestManifest(testStatefulSet *appsv1.StatefulSet, withVolume bool) { +func readStatefulSetTestManifest(testStatefulSet *appsv1.StatefulSet, variant StatefulSetVariant) { By("Read the statefulSet resource") - if withVolume { - Expect(utils.GetObjectFromManifest("resources/statefulset-with-volume.yaml", testStatefulSet)).Should(Succeed()) - } else { - Expect(utils.GetObjectFromManifest("resources/test-statefulset.yaml", testStatefulSet)).Should(Succeed()) - } + var manifestPath string + switch variant { + case StatefulSetBasic: + manifestPath = "resources/statefulset-basic.yaml" + case StatefulSetInvalidStorage: + manifestPath = "resources/statefulset-invalid-storage.yaml" + case StatefulSetWithStorage: + manifestPath = "resources/statefulset-with-storage.yaml" + } + Expect(utils.GetObjectFromManifest(manifestPath, testStatefulSet)).Should(Succeed()) } func readServiceTestManifest(testService *corev1.Service) { From d5fcf88ff6e2728deb6350cdf4559bb6c3714954 Mon Sep 17 00:00:00 2001 From: Wei Weng Date: Mon, 1 Dec 2025 15:45:24 -0800 Subject: [PATCH 03/16] feat: publish image to ghcr with tag (#353) * publish image with tag Signed-off-by: Wei Weng * fix unknown/unknown Signed-off-by: Wei Weng * comment Signed-off-by: Wei Weng * commit suggestions Signed-off-by: Wei Weng * remove provenance=false Signed-off-by: Wei Weng --------- Signed-off-by: Wei Weng Co-authored-by: Wei Weng --- .github/workflows/release.yml | 87 +++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..57227d1a9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,87 @@ +name: Release Images + +on: + push: + tags: + - 'v*.*.*' + workflow_dispatch: + inputs: + tag: + description: 'Release tag (e.g., v1.0.0)' + required: true + type: string + +permissions: + contents: read + packages: write + +env: + REGISTRY: ghcr.io + HUB_AGENT_IMAGE_NAME: hub-agent + MEMBER_AGENT_IMAGE_NAME: member-agent + REFRESH_TOKEN_IMAGE_NAME: refresh-token + GO_VERSION: '1.24.9' + +jobs: + export-registry: + runs-on: ubuntu-latest + outputs: + registry: ${{ steps.export.outputs.registry }} + tag: ${{ steps.export.outputs.tag }} + steps: + - name: Checkout code + uses: actions/checkout@v6.0.0 + + - id: export + run: | + # registry must be in lowercase + echo "registry=$(echo "${{ env.REGISTRY }}/${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT + + # Extract tag from github ref or workflow input + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + TAG="${{ inputs.tag }}" + elif [[ "${{ github.ref }}" == refs/tags/* ]]; then + TAG=${GITHUB_REF#refs/tags/} + else + echo "Error: Workflow triggered by unsupported event or ref" + echo "Event: ${{ github.event_name }}" + echo "Ref: ${{ github.ref }}" + exit 1 + fi + echo "tag=${TAG}" >> $GITHUB_OUTPUT + echo "Release tag: ${TAG}" + + build-and-publish: + needs: export-registry + env: + REGISTRY: ${{ needs.export-registry.outputs.registry }} + TAG: ${{ needs.export-registry.outputs.tag }} + runs-on: ubuntu-latest + steps: + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Checkout code + uses: actions/checkout@v6.0.0 + + - name: Login to ghcr.io + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push images with tag ${{ env.TAG }} + run: | + make push + + - name: Verify images + run: | + echo "✅ Published images:" + echo " - ${{ env.REGISTRY }}/${{ env.HUB_AGENT_IMAGE_NAME }}:${{ env.TAG }}" + echo " - ${{ env.REGISTRY }}/${{ env.MEMBER_AGENT_IMAGE_NAME }}:${{ env.TAG }}" + echo " - ${{ env.REGISTRY }}/${{ env.REFRESH_TOKEN_IMAGE_NAME }}:${{ env.TAG }}" + echo "" + echo "📦 Images are now public!" From 0376725d468dabae3e7a0dfbaa3e8133cf9806eb Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Tue, 2 Dec 2025 23:43:48 +0800 Subject: [PATCH 04/16] chore: bump controller runtime version to v0.22.x (#354) * Bump controller runtime version Signed-off-by: michaelawyu * Minor fixes Signed-off-by: michaelawyu --------- Signed-off-by: michaelawyu --- go.mod | 6 +++--- go.sum | 12 ++++++------ .../v1beta1/member_controller_test.go | 2 +- .../v1beta1/membercluster_controller_test.go | 2 +- .../fleetresourcehandler_webhook_test.go | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index d5eb73fbe..704b41915 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 github.com/Azure/karpenter-provider-azure v1.5.1 - github.com/crossplane/crossplane-runtime v1.20.0 + github.com/crossplane/crossplane-runtime/v2 v2.1.0 github.com/evanphx/json-patch/v5 v5.9.11 github.com/google/go-cmp v0.7.0 github.com/onsi/ginkgo/v2 v2.23.4 @@ -37,11 +37,11 @@ require ( sigs.k8s.io/cloud-provider-azure v1.32.4 sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.5.20 sigs.k8s.io/cluster-inventory-api v0.0.0-20251028164203-2e3fabb46733 - sigs.k8s.io/controller-runtime v0.21.0 + sigs.k8s.io/controller-runtime v0.22.4 ) require ( - dario.cat/mergo v1.0.1 // indirect + dario.cat/mergo v1.0.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.4.0 // indirect diff --git a/go.sum b/go.sum index 5bb3b922e..1801eb133 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= -dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/Azure/aks-middleware v0.0.40 h1:eFRuAxCcIAZoy/6+FvumDl2KOWnSPxXcAeCSOA4+aTo= github.com/Azure/aks-middleware v0.0.40/go.mod h1:7Y+wxZmS7p1K0FPreiO3+6Wr8YhYjWz9c50YohDQIQ4= github.com/Azure/azure-kusto-go v0.16.1 h1:vCBWcQghmC1qIErUUgVNWHxGhZVStu1U/hki6iBA14k= @@ -97,8 +97,8 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/crossplane/crossplane-runtime v1.20.0 h1:I54uipRIecqZyms+vz1J/l62yjVQ7HV5w+Nh3RMrUtc= -github.com/crossplane/crossplane-runtime v1.20.0/go.mod h1:lfV1VJenDc9PNVLxDC80YjPoTm+JdSZ13xlS2h37Dvg= +github.com/crossplane/crossplane-runtime/v2 v2.1.0 h1:JBMhL9T+/PfyjLAQEdZWlKLvA3jJVtza8zLLwd9Gs4k= +github.com/crossplane/crossplane-runtime/v2 v2.1.0/go.mod h1:j78pmk0qlI//Ur7zHhqTr8iePHFcwJKrZnzZB+Fg4t0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -425,8 +425,8 @@ sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.5.2 h1:jjFJF0PmS9I sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.5.2/go.mod h1:7DdZ9ipIsmPLpBlfT4gueejcUlJBZQKWhdljQE5SKvc= sigs.k8s.io/cluster-inventory-api v0.0.0-20251028164203-2e3fabb46733 h1:l90ANqblqFrE4L2QLLk+9iPjfmaLRvOFL51l/fgwUgg= sigs.k8s.io/cluster-inventory-api v0.0.0-20251028164203-2e3fabb46733/go.mod h1:guwenlZ9iIfYlNxn7ExCfugOLTh6wjjRX3adC36YCmQ= -sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= -sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= +sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= +sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/karpenter v1.5.0 h1:3HaFtFvkteUJ+SjIViR1ImR0qR+GTqDulahauIuE4Qg= diff --git a/pkg/controllers/internalmembercluster/v1beta1/member_controller_test.go b/pkg/controllers/internalmembercluster/v1beta1/member_controller_test.go index b95a3a0b4..085be2b28 100644 --- a/pkg/controllers/internalmembercluster/v1beta1/member_controller_test.go +++ b/pkg/controllers/internalmembercluster/v1beta1/member_controller_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/crossplane-runtime/v2/pkg/test" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" diff --git a/pkg/controllers/membercluster/v1beta1/membercluster_controller_test.go b/pkg/controllers/membercluster/v1beta1/membercluster_controller_test.go index 9bf170e63..54dffbe9e 100644 --- a/pkg/controllers/membercluster/v1beta1/membercluster_controller_test.go +++ b/pkg/controllers/membercluster/v1beta1/membercluster_controller_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/crossplane-runtime/v2/pkg/test" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" diff --git a/pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook_test.go b/pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook_test.go index a0c14b9d6..9c64c7fff 100644 --- a/pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook_test.go +++ b/pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook_test.go @@ -7,7 +7,7 @@ import ( "fmt" "testing" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/crossplane-runtime/v2/pkg/test" "github.com/stretchr/testify/assert" admissionv1 "k8s.io/api/admission/v1" authenticationv1 "k8s.io/api/authentication/v1" From 2e50010509ba5fab288e4c4bb4a45415a01d655f Mon Sep 17 00:00:00 2001 From: Britania Rodriguez Reyes <145056127+britaniar@users.noreply.github.com> Date: Tue, 2 Dec 2025 10:55:15 -0600 Subject: [PATCH 05/16] feat: add beforeStageTasks to stagedUpdateRun (#328) * add beforeStageTasks to stagedUpdateRun Signed-off-by: Britania Rodriguez Reyes --- apis/placement/v1beta1/commons.go | 7 +- pkg/controllers/updaterun/controller.go | 9 +- .../updaterun/controller_integration_test.go | 23 +- pkg/controllers/updaterun/controller_test.go | 61 +++- pkg/controllers/updaterun/execution.go | 224 ++++++++----- .../updaterun/execution_integration_test.go | 311 +++++++++++++++--- pkg/controllers/updaterun/execution_test.go | 254 ++++++++++++++ pkg/controllers/updaterun/initialization.go | 39 ++- .../initialization_integration_test.go | 76 +++-- .../updaterun/initialization_test.go | 108 +++++- .../updaterun/validation_integration_test.go | 16 +- pkg/utils/condition/reason.go | 11 +- test/e2e/actuals_test.go | 18 +- test/e2e/cluster_staged_updaterun_test.go | 251 +++++++++----- test/e2e/staged_updaterun_test.go | 156 +++++++-- 15 files changed, 1271 insertions(+), 293 deletions(-) diff --git a/apis/placement/v1beta1/commons.go b/apis/placement/v1beta1/commons.go index 03e8e9dc0..9ced5c1b5 100644 --- a/apis/placement/v1beta1/commons.go +++ b/apis/placement/v1beta1/commons.go @@ -167,8 +167,11 @@ const ( // TargetUpdatingStageNameLabel indicates the updating stage name on a staged run related object. TargetUpdatingStageNameLabel = FleetPrefix + "targetUpdatingStage" - // ApprovalTaskNameFmt is the format of the approval task name. - ApprovalTaskNameFmt = "%s-%s" + // BeforeStageApprovalTaskNameFmt is the format of the before stage approval task name. + BeforeStageApprovalTaskNameFmt = "%s-before-%s" + + // AfterStageApprovalTaskNameFmt is the format of the after stage approval task name. + AfterStageApprovalTaskNameFmt = "%s-after-%s" ) var ( diff --git a/pkg/controllers/updaterun/controller.go b/pkg/controllers/updaterun/controller.go index 9a0ee4862..4e8fa695c 100644 --- a/pkg/controllers/updaterun/controller.go +++ b/pkg/controllers/updaterun/controller.go @@ -79,7 +79,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim } runObjRef := klog.KObj(updateRun) - // Remove waitTime from the updateRun status for AfterStageTask for type Approval. + // Remove waitTime from the updateRun status for BeforeStageTask and AfterStageTask for type Approval. removeWaitTimeFromUpdateRunStatus(updateRun) // Handle the deletion of the updateRun. @@ -455,10 +455,15 @@ func emitUpdateRunStatusMetric(updateRun placementv1beta1.UpdateRunObj) { } func removeWaitTimeFromUpdateRunStatus(updateRun placementv1beta1.UpdateRunObj) { - // Remove waitTime from the updateRun status for AfterStageTask for type Approval. + // Remove waitTime from the updateRun status for BeforeStageTask and AfterStageTask for type Approval. updateRunStatus := updateRun.GetUpdateRunStatus() if updateRunStatus.UpdateStrategySnapshot != nil { for i := range updateRunStatus.UpdateStrategySnapshot.Stages { + for j := range updateRunStatus.UpdateStrategySnapshot.Stages[i].BeforeStageTasks { + if updateRunStatus.UpdateStrategySnapshot.Stages[i].BeforeStageTasks[j].Type == placementv1beta1.StageTaskTypeApproval { + updateRunStatus.UpdateStrategySnapshot.Stages[i].BeforeStageTasks[j].WaitTime = nil + } + } for j := range updateRunStatus.UpdateStrategySnapshot.Stages[i].AfterStageTasks { if updateRunStatus.UpdateStrategySnapshot.Stages[i].AfterStageTasks[j].Type == placementv1beta1.StageTaskTypeApproval { updateRunStatus.UpdateStrategySnapshot.Stages[i].AfterStageTasks[j].WaitTime = nil diff --git a/pkg/controllers/updaterun/controller_integration_test.go b/pkg/controllers/updaterun/controller_integration_test.go index 8c8e38d06..d33b132b3 100644 --- a/pkg/controllers/updaterun/controller_integration_test.go +++ b/pkg/controllers/updaterun/controller_integration_test.go @@ -505,6 +505,11 @@ func generateTestClusterStagedUpdateStrategy() *placementv1beta1.ClusterStagedUp }, }, SortingLabelKey: &sortingKey, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, AfterStageTasks: []placementv1beta1.StageTask{ { Type: placementv1beta1.StageTaskTypeTimedWait, @@ -526,6 +531,11 @@ func generateTestClusterStagedUpdateStrategy() *placementv1beta1.ClusterStagedUp }, }, // no sortingLabelKey, should sort by cluster name + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, AfterStageTasks: []placementv1beta1.StageTask{ { Type: placementv1beta1.StageTaskTypeApproval, @@ -543,7 +553,7 @@ func generateTestClusterStagedUpdateStrategy() *placementv1beta1.ClusterStagedUp } } -func generateTestClusterStagedUpdateStrategyWithSingleStage(afterStageTasks []placementv1beta1.StageTask) *placementv1beta1.ClusterStagedUpdateStrategy { +func generateTestClusterStagedUpdateStrategyWithSingleStage(beforeStageTasks, afterStageTasks []placementv1beta1.StageTask) *placementv1beta1.ClusterStagedUpdateStrategy { return &placementv1beta1.ClusterStagedUpdateStrategy{ ObjectMeta: metav1.ObjectMeta{ Name: testUpdateStrategyName, @@ -551,9 +561,10 @@ func generateTestClusterStagedUpdateStrategyWithSingleStage(afterStageTasks []pl Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { - Name: "stage1", - LabelSelector: &metav1.LabelSelector{}, // Select all clusters. - AfterStageTasks: afterStageTasks, + Name: "stage1", + LabelSelector: &metav1.LabelSelector{}, // Select all clusters. + BeforeStageTasks: beforeStageTasks, + AfterStageTasks: afterStageTasks, }, }, }, @@ -724,9 +735,9 @@ func generateTrueCondition(obj client.Object, condType any) metav1.Condition { case placementv1beta1.StageTaskConditionWaitTimeElapsed: reason = condition.AfterStageTaskWaitTimeElapsedReason case placementv1beta1.StageTaskConditionApprovalRequestCreated: - reason = condition.AfterStageTaskApprovalRequestCreatedReason + reason = condition.StageTaskApprovalRequestCreatedReason case placementv1beta1.StageTaskConditionApprovalRequestApproved: - reason = condition.AfterStageTaskApprovalRequestApprovedReason + reason = condition.StageTaskApprovalRequestApprovedReason } typeStr = string(cond) case placementv1beta1.ApprovalRequestConditionType: diff --git a/pkg/controllers/updaterun/controller_test.go b/pkg/controllers/updaterun/controller_test.go index f842960f2..7fe56913f 100644 --- a/pkg/controllers/updaterun/controller_test.go +++ b/pkg/controllers/updaterun/controller_test.go @@ -912,7 +912,7 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }, }, }, - "should remove waitTime from Approval tasks only": { + "should remove waitTime from Approval tasks only for AfterStageTasks": { inputUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ Status: placementv1beta1.UpdateRunStatus{ UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ @@ -953,12 +953,51 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }, }, }, + "should remove waitTime from Approval tasks only for BeforeStageTasks": { + inputUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + WaitTime: &waitTime, + }, + }, + }, + }, + }, + }, + }, + wantUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + }, + }, + }, + }, + }, + }, "should handle multiple stages": { inputUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ Status: placementv1beta1.UpdateRunStatus{ UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + WaitTime: &waitTime, + }, + }, AfterStageTasks: []placementv1beta1.StageTask{ { Type: placementv1beta1.StageTaskTypeApproval, @@ -978,6 +1017,14 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }, }, }, + { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + WaitTime: &waitTime, + }, + }, + }, }, }, }, @@ -987,6 +1034,11 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, AfterStageTasks: []placementv1beta1.StageTask{ { Type: placementv1beta1.StageTaskTypeApproval, @@ -1004,6 +1056,13 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }, }, }, + { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + }, }, }, }, diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index 198980749..074c23739 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -62,34 +62,78 @@ func (r *Reconciler) execute( updateRun placementv1beta1.UpdateRunObj, updatingStageIndex int, toBeUpdatedBindings, toBeDeletedBindings []placementv1beta1.BindingObj, -) (bool, time.Duration, error) { +) (finished bool, waitTime time.Duration, err error) { + updateRunStatus := updateRun.GetUpdateRunStatus() + var updatingStageStatus *placementv1beta1.StageUpdatingStatus + + // Set up defer function to handle errStagedUpdatedAborted. + defer func() { + if errors.Is(err, errStagedUpdatedAborted) { + if updatingStageStatus != nil { + markStageUpdatingFailed(updatingStageStatus, updateRun.GetGeneration(), err.Error()) + } else { + // Handle deletion stage case. + markStageUpdatingFailed(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration(), err.Error()) + } + } + }() + // Mark updateRun as progressing if it's not already marked as waiting or stuck. // This avoids triggering an unnecessary in-memory transition from stuck (waiting) -> progressing -> stuck (waiting), // which would update the lastTransitionTime even though the status hasn't effectively changed. markUpdateRunProgressingIfNotWaitingOrStuck(updateRun) - - updateRunStatus := updateRun.GetUpdateRunStatus() if updatingStageIndex < len(updateRunStatus.StagesStatus) { maxConcurrency, err := calculateMaxConcurrencyValue(updateRunStatus, updatingStageIndex) if err != nil { return false, 0, err } - updatingStage := &updateRunStatus.StagesStatus[updatingStageIndex] - waitTime, execErr := r.executeUpdatingStage(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, maxConcurrency) - if errors.Is(execErr, errStagedUpdatedAborted) { - markStageUpdatingFailed(updatingStage, updateRun.GetGeneration(), execErr.Error()) - return true, waitTime, execErr + updatingStageStatus = &updateRunStatus.StagesStatus[updatingStageIndex] + approved, err := r.checkBeforeStageTasksStatus(ctx, updatingStageIndex, updateRun) + if err != nil { + return false, 0, err } + if !approved { + markStageUpdatingWaiting(updatingStageStatus, updateRun.GetGeneration(), "Not all before-stage tasks are completed, waiting for approval") + markUpdateRunWaiting(updateRun, fmt.Sprintf(condition.UpdateRunWaitingMessageFmt, "before-stage", updatingStageStatus.StageName)) + return false, stageUpdatingWaitTime, nil + } + waitTime, err = r.executeUpdatingStage(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, maxConcurrency) // The execution has not finished yet. - return false, waitTime, execErr + return false, waitTime, err } // All the stages have finished, now start the delete stage. - finished, execErr := r.executeDeleteStage(ctx, updateRun, toBeDeletedBindings) - if errors.Is(execErr, errStagedUpdatedAborted) { - markStageUpdatingFailed(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration(), execErr.Error()) - return true, 0, execErr + finished, err = r.executeDeleteStage(ctx, updateRun, toBeDeletedBindings) + return finished, clusterUpdatingWaitTime, err +} + +// checkBeforeStageTasksStatus checks if the before stage tasks have finished. +// It returns if the before stage tasks have finished or error if the before stage tasks failed. +func (r *Reconciler) checkBeforeStageTasksStatus(ctx context.Context, updatingStageIndex int, updateRun placementv1beta1.UpdateRunObj) (bool, error) { + updateRunRef := klog.KObj(updateRun) + updateRunStatus := updateRun.GetUpdateRunStatus() + updatingStageStatus := &updateRunStatus.StagesStatus[updatingStageIndex] + updatingStage := &updateRunStatus.UpdateStrategySnapshot.Stages[updatingStageIndex] + if updatingStage.BeforeStageTasks == nil { + klog.V(2).InfoS("There is no before stage task for this stage", "stage", updatingStage.Name, "updateRun", updateRunRef) + return true, nil } - return finished, clusterUpdatingWaitTime, execErr + + for i, task := range updatingStage.BeforeStageTasks { + switch task.Type { + case placementv1beta1.StageTaskTypeApproval: + approved, err := r.handleStageApprovalTask(ctx, &updatingStageStatus.BeforeStageTaskStatus[i], updatingStage, updateRun) + if err != nil { + return false, err + } + return approved, nil // Ideally there should be only one approval task in before stage tasks. + default: + // Approval is the only supported before stage task. + unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("found unsupported task type in before stage tasks: %s", task.Type)) + klog.ErrorS(unexpectedErr, "Task type is not supported in before stage tasks", "stage", updatingStage.Name, "updateRun", updateRunRef, "taskType", task.Type) + return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) + } + } + return true, nil } // executeUpdatingStage executes a single updating stage by updating the bindings. @@ -242,8 +286,8 @@ func (r *Reconciler) executeUpdatingStage( if finishedClusterCount == len(updatingStageStatus.Clusters) { // All the clusters in the stage have been updated. - markUpdateRunWaiting(updateRun, updatingStageStatus.StageName) - markStageUpdatingWaiting(updatingStageStatus, updateRun.GetGeneration()) + markUpdateRunWaiting(updateRun, fmt.Sprintf(condition.UpdateRunWaitingMessageFmt, "after-stage", updatingStageStatus.StageName)) + markStageUpdatingWaiting(updatingStageStatus, updateRun.GetGeneration(), "All clusters in the stage are updated, waiting for after-stage tasks to complete") klog.V(2).InfoS("The stage has finished all cluster updating", "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) // Check if the after stage tasks are ready. approved, waitTime, err := r.checkAfterStageTasksStatus(ctx, updatingStageIndex, updateRun) @@ -360,59 +404,11 @@ func (r *Reconciler) checkAfterStageTasksStatus(ctx context.Context, updatingSta klog.V(2).InfoS("The after stage wait task has completed", "stage", updatingStage.Name, "updateRun", updateRunRef) } case placementv1beta1.StageTaskTypeApproval: - afterStageTaskApproved := condition.IsConditionStatusTrue(meta.FindStatusCondition(updatingStageStatus.AfterStageTaskStatus[i].Conditions, string(placementv1beta1.StageTaskConditionApprovalRequestApproved)), updateRun.GetGeneration()) - if afterStageTaskApproved { - // The afterStageTask has been approved. - continue + approved, err := r.handleStageApprovalTask(ctx, &updatingStageStatus.AfterStageTaskStatus[i], updatingStage, updateRun) + if err != nil { + return false, -1, err } - // Check if the approval request has been created. - approvalRequest := buildApprovalRequestObject(types.NamespacedName{Name: updatingStageStatus.AfterStageTaskStatus[i].ApprovalRequestName, Namespace: updateRun.GetNamespace()}, updatingStage.Name, updateRun.GetName()) - requestRef := klog.KObj(approvalRequest) - if err := r.Client.Create(ctx, approvalRequest); err != nil { - if apierrors.IsAlreadyExists(err) { - // The approval task already exists. - markAfterStageRequestCreated(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.GetGeneration()) - if err = r.Client.Get(ctx, client.ObjectKeyFromObject(approvalRequest), approvalRequest); err != nil { - klog.ErrorS(err, "Failed to get the already existing approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - return false, -1, controller.NewAPIServerError(true, err) - } - approvalRequestSpec := approvalRequest.GetApprovalRequestSpec() - if approvalRequestSpec.TargetStage != updatingStage.Name || approvalRequestSpec.TargetUpdateRun != updateRun.GetName() { - unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the approval request task `%s/%s` is targeting update run `%s/%s` and stage `%s`", approvalRequest.GetNamespace(), approvalRequest.GetName(), approvalRequest.GetNamespace(), approvalRequestSpec.TargetUpdateRun, approvalRequestSpec.TargetStage)) - klog.ErrorS(unexpectedErr, "Found an approval request targeting wrong stage", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - return false, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) - } - approvalRequestStatus := approvalRequest.GetApprovalRequestStatus() - approvalAccepted := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequestStatus.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.GetGeneration()) - approved := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequestStatus.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)), approvalRequest.GetGeneration()) - if !approvalAccepted && !approved { - klog.V(2).InfoS("The approval request has not been approved yet", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - passed = false - continue - } - if approved { - klog.V(2).InfoS("The approval request has been approved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - if !approvalAccepted { - if err = r.updateApprovalRequestAccepted(ctx, approvalRequest); err != nil { - klog.ErrorS(err, "Failed to accept the approved approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - // retriable err - return false, -1, err - } - } - } else { - // Approved state should not change once the approval is accepted. - klog.V(2).InfoS("The approval request has been approval-accepted, ignoring changing back to unapproved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - } - markAfterStageRequestApproved(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.GetGeneration()) - } else { - // retriable error - klog.ErrorS(err, "Failed to create the approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - return false, -1, controller.NewAPIServerError(false, err) - } - } else { - // The approval request has been created for the first time. - klog.V(2).InfoS("The approval request has been created", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - markAfterStageRequestCreated(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.GetGeneration()) + if !approved { passed = false } } @@ -423,6 +419,74 @@ func (r *Reconciler) checkAfterStageTasksStatus(ctx context.Context, updatingSta return passed, afterStageWaitTime, nil } +// handleStageApprovalTask handles the approval task logic for before or after stage tasks. +// It returns true if the task is approved, false otherwise, and any error encountered. +func (r *Reconciler) handleStageApprovalTask( + ctx context.Context, + stageTaskStatus *placementv1beta1.StageTaskStatus, + updatingStage *placementv1beta1.StageConfig, + updateRun placementv1beta1.UpdateRunObj, +) (bool, error) { + updateRunRef := klog.KObj(updateRun) + + stageTaskApproved := condition.IsConditionStatusTrue(meta.FindStatusCondition(stageTaskStatus.Conditions, string(placementv1beta1.StageTaskConditionApprovalRequestApproved)), updateRun.GetGeneration()) + if stageTaskApproved { + // The stageTask has been approved. + return true, nil + } + + // Check if the approval request has been created. + approvalRequest := buildApprovalRequestObject(types.NamespacedName{Name: stageTaskStatus.ApprovalRequestName, Namespace: updateRun.GetNamespace()}, updatingStage.Name, updateRun.GetName()) + requestRef := klog.KObj(approvalRequest) + if err := r.Client.Create(ctx, approvalRequest); err != nil { + if apierrors.IsAlreadyExists(err) { + // The approval task already exists. + markStageTaskRequestCreated(stageTaskStatus, updateRun.GetGeneration()) + if err = r.Client.Get(ctx, client.ObjectKeyFromObject(approvalRequest), approvalRequest); err != nil { + klog.ErrorS(err, "Failed to get the already existing approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + return false, controller.NewAPIServerError(true, err) + } + approvalRequestSpec := approvalRequest.GetApprovalRequestSpec() + if approvalRequestSpec.TargetStage != updatingStage.Name || approvalRequestSpec.TargetUpdateRun != updateRun.GetName() { + unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the approval request task `%s/%s` is targeting update run `%s/%s` and stage `%s`, want target update run `%s/%s and stage `%s`", approvalRequest.GetNamespace(), approvalRequest.GetName(), approvalRequest.GetNamespace(), approvalRequestSpec.TargetUpdateRun, approvalRequestSpec.TargetStage, approvalRequest.GetNamespace(), updateRun.GetName(), updatingStage.Name)) + klog.ErrorS(unexpectedErr, "Found an approval request targeting wrong stage", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) + } + approvalRequestStatus := approvalRequest.GetApprovalRequestStatus() + approvalAccepted := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequestStatus.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.GetGeneration()) + approved := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequestStatus.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)), approvalRequest.GetGeneration()) + if !approvalAccepted && !approved { + klog.V(2).InfoS("The approval request has not been approved yet", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + return false, nil + } + if approved { + klog.V(2).InfoS("The approval request has been approved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + if !approvalAccepted { + if err = r.updateApprovalRequestAccepted(ctx, approvalRequest); err != nil { + klog.ErrorS(err, "Failed to accept the approved approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + // retriable err + return false, err + } + } + } else { + // Approved state should not change once the approval is accepted. + klog.V(2).InfoS("The approval request has been approval-accepted, ignoring changing back to unapproved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + } + markStageTaskRequestApproved(stageTaskStatus, updateRun.GetGeneration()) + } else { + // retriable error + klog.ErrorS(err, "Failed to create the approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + return false, controller.NewAPIServerError(false, err) + } + } else { + // The approval request has been created for the first time. + klog.V(2).InfoS("The approval request has been created", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + markStageTaskRequestCreated(stageTaskStatus, updateRun.GetGeneration()) + return false, nil + } + return true, nil +} + // updateBindingRolloutStarted updates the binding status to indicate the rollout has started. func (r *Reconciler) updateBindingRolloutStarted(ctx context.Context, binding placementv1beta1.BindingObj, updateRun placementv1beta1.UpdateRunObj) error { // first reset the condition to reflect the latest lastTransitionTime @@ -616,14 +680,14 @@ func markUpdateRunStuck(updateRun placementv1beta1.UpdateRunObj, stageName, clus } // markUpdateRunWaiting marks the updateRun as waiting in memory. -func markUpdateRunWaiting(updateRun placementv1beta1.UpdateRunObj, stageName string) { +func markUpdateRunWaiting(updateRun placementv1beta1.UpdateRunObj, message string) { updateRunStatus := updateRun.GetUpdateRunStatus() meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), Status: metav1.ConditionFalse, ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunWaitingReason, - Message: fmt.Sprintf("The updateRun is waiting for after-stage tasks in stage %s to complete", stageName), + Message: message, }) } @@ -642,13 +706,13 @@ func markStageUpdatingStarted(stageUpdatingStatus *placementv1beta1.StageUpdatin } // markStageUpdatingWaiting marks the stage updating status as waiting in memory. -func markStageUpdatingWaiting(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64) { +func markStageUpdatingWaiting(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64, message string) { meta.SetStatusCondition(&stageUpdatingStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StageUpdatingConditionProgressing), Status: metav1.ConditionFalse, ObservedGeneration: generation, Reason: condition.StageUpdatingWaitingReason, - Message: "All clusters in the stage are updated, waiting for after-stage tasks to complete", + Message: message, }) } @@ -727,24 +791,24 @@ func markClusterUpdatingFailed(clusterUpdatingStatus *placementv1beta1.ClusterUp }) } -// markAfterStageRequestCreated marks the Approval after stage task as ApprovalRequestCreated in memory. -func markAfterStageRequestCreated(afterStageTaskStatus *placementv1beta1.StageTaskStatus, generation int64) { - meta.SetStatusCondition(&afterStageTaskStatus.Conditions, metav1.Condition{ +// markStageTaskRequestCreated marks the Approval for the before or after stage task as ApprovalRequestCreated in memory. +func markStageTaskRequestCreated(stageTaskStatus *placementv1beta1.StageTaskStatus, generation int64) { + meta.SetStatusCondition(&stageTaskStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StageTaskConditionApprovalRequestCreated), Status: metav1.ConditionTrue, ObservedGeneration: generation, - Reason: condition.AfterStageTaskApprovalRequestCreatedReason, + Reason: condition.StageTaskApprovalRequestCreatedReason, Message: "ApprovalRequest object is created", }) } -// markAfterStageRequestApproved marks the Approval after stage task as Approved in memory. -func markAfterStageRequestApproved(afterStageTaskStatus *placementv1beta1.StageTaskStatus, generation int64) { - meta.SetStatusCondition(&afterStageTaskStatus.Conditions, metav1.Condition{ +// markStageTaskRequestApproved marks the Approval for the before or after stage task as Approved in memory. +func markStageTaskRequestApproved(stageTaskStatus *placementv1beta1.StageTaskStatus, generation int64) { + meta.SetStatusCondition(&stageTaskStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StageTaskConditionApprovalRequestApproved), Status: metav1.ConditionTrue, ObservedGeneration: generation, - Reason: condition.AfterStageTaskApprovalRequestApprovedReason, + Reason: condition.StageTaskApprovalRequestApprovedReason, Message: "ApprovalRequest object is approved", }) } diff --git a/pkg/controllers/updaterun/execution_integration_test.go b/pkg/controllers/updaterun/execution_integration_test.go index 980283497..61764f378 100644 --- a/pkg/controllers/updaterun/execution_integration_test.go +++ b/pkg/controllers/updaterun/execution_integration_test.go @@ -155,17 +155,64 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { }) Context("Cluster staged update run should update clusters one by one", Ordered, func() { + var wantApprovalRequest *placementv1beta1.ClusterApprovalRequest BeforeAll(func() { By("Creating a new clusterStagedUpdateRun") Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) - By("Validating the initialization succeeded and the execution started") + By("Validating the initialization succeeded and the execution has not started") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - wantStatus = generateExecutionStartedStatus(updateRun, initialized) + wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + By("Validating the first beforeStage approvalRequest has been created") + wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[0].BeforeStageTaskStatus[0].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[0].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) + }) + + It("Should not start rolling out 1st stage", func() { + By("Validating the 1st clusterResourceBinding is not updated to Bound") + binding := resourceBindings[numTargetClusters-1] // cluster-9 + validateNotBoundBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Validating the 1st stage does not have startTime set") + Expect(updateRun.Status.StagesStatus[0].StartTime).Should(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) + }) + + It("Should accept the approval request and start to rollout 1st stage", func() { + By("Approving the approvalRequest") + approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) + + By("Validating the approvalRequest has ApprovalAccepted status") + Eventually(func() (bool, error) { + var approvalRequest placementv1beta1.ClusterApprovalRequest + if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { + return false, err + } + return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil + }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + // Approval task has been approved. + wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) }) It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -177,6 +224,9 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + // 1st stage started. + wantStatus = generateExecutionStartedStatus(updateRun, wantStatus) + By("Validating the 1st cluster has succeeded and 2nd cluster has started") wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) @@ -186,7 +236,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 2nd cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -204,7 +254,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 3rd cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -222,7 +272,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 4th cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -240,7 +290,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 5th cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -252,12 +302,13 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - By("Validating the 5th cluster has succeeded and stage waiting for AfterStageTasks") + By("Validating the 5th cluster has succeeded and 1st stage has completed and is waiting for AfterStageTasks") + // 5th cluster succeeded. wantStatus.StagesStatus[0].Clusters[4].Conditions = append(wantStatus.StagesStatus[0].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[0].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. - wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, - generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + // Now waiting for after stage tasks of 1st stage. + meta.SetStatusCondition(&wantStatus.StagesStatus[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") @@ -295,11 +346,11 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { // 1st stage completed, mark progressing condition reason as succeeded and add succeeded condition. wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) - // 2nd stage started. - wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) - // 1st cluster in 2nd stage started. - wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - wantStatus.Conditions[1] = generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + // 2nd stage waiting for before stage tasks. + wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Validating the 1st stage has endTime set") @@ -316,10 +367,61 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Expect(approvalCreateTime.Before(waitEndTime)).Should(BeTrue()) By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) + }) + + It("Should create approval request before 2nd stage", func() { + By("Validating the approvalRequest has been created") + wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[1].BeforeStageTaskStatus[0].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[1].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[1].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) }) - It("Should mark the 1st cluster in the 2nd stage as succeeded after marking the binding available", func() { + It("Should not start rolling out 2nd stage", func() { + By("Validating the 1st clusterResourceBinding is not updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBoundBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) + + By("Validating the 1st stage does not have startTime set") + Expect(updateRun.Status.StagesStatus[1].StartTime).Should(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) + }) + + It("Should accept the approval request and start to rollout 2nd stage", func() { + By("Approving the approvalRequest") + approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) + + By("Validating the approvalRequest has ApprovalAccepted status") + Eventually(func() (bool, error) { + var approvalRequest placementv1beta1.ClusterApprovalRequest + if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { + return false, err + } + return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil + }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + // Approval task has been approved. + wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) + }) + + It("Should mark the 1st cluster in the 2nd stage as succeeded after approving request and marking the binding available", func() { By("Validating the 1st clusterResourceBinding is updated to Bound") binding := resourceBindings[0] // cluster-0 validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) @@ -327,6 +429,11 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { By("Updating the 1st clusterResourceBinding to Available") meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + // 2nd stage started. + wantStatus.StagesStatus[1].Conditions[0] = generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + // 1st cluster started. + wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) By("Validating the 1st cluster has succeeded and 2nd cluster has started") wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) @@ -408,7 +515,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { wantStatus.StagesStatus[1].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") @@ -443,7 +550,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) wantStatus.StagesStatus[1].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) - wantStatus.Conditions[1] = generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) for i := range wantStatus.DeletionStageStatus.Clusters { @@ -501,7 +608,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { wantStatus.DeletionStageStatus.Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -520,13 +627,13 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { By("Creating a new clusterStagedUpdateRun") Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) - By("Validating the initialization succeeded and the execution started") + By("Validating the initialization succeeded and the execution has not started") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - wantStatus = generateExecutionStartedStatus(updateRun, initialized) + wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) }) AfterAll(func() { @@ -535,6 +642,24 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { }) It("Should keep waiting for the 1st cluster while it's not available", func() { + By("Approving the approvalRequest") + approvalName := updateRun.Status.StagesStatus[0].BeforeStageTaskStatus[0].ApprovalRequestName + approveClusterApprovalRequest(ctx, approvalName) + + By("Validating the approvalRequest has ApprovalAccepted status") + Eventually(func() (bool, error) { + var approvalRequest placementv1beta1.ClusterApprovalRequest + if err := k8sClient.Get(ctx, types.NamespacedName{Name: approvalName}, &approvalRequest); err != nil { + return false, err + } + return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil + }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + // Approval task has been approved. + wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) + // 1st stage started. + wantStatus = generateExecutionStartedStatus(updateRun, wantStatus) + By("Validating the 1st clusterResourceBinding is updated to Bound") binding := resourceBindings[numTargetClusters-1] // cluster-9 validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) @@ -544,7 +669,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") By("Validating the updateRun is stuck in the 1st cluster of the 1st stage") - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) wantStatus.Conditions[1].Reason = condition.UpdateRunStuckReason validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, wantStatus, "") @@ -552,7 +677,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { It("Should emit stuck status metrics after time waiting for the 1st cluster reaches threshold", func() { By("Checking update run stuck metrics is emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateStuckMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun), generateStuckMetric(updateRun)) }) It("Should abort the execution if the binding has unexpected state", func() { @@ -568,12 +693,12 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, false) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, false) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, false)) wantStatus.Conditions = append(wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateStuckMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun), generateStuckMetric(updateRun), generateFailedMetric(updateRun)) }) }) }) @@ -602,7 +727,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { policySnapshot = generateTestClusterSchedulingPolicySnapshot(1, len(targetClusters)) resourceSnapshot = generateTestClusterResourceSnapshot() resourceSnapshot = generateTestClusterResourceSnapshot() - updateStrategy = generateTestClusterStagedUpdateStrategyWithSingleStage(nil) + updateStrategy = generateTestClusterStagedUpdateStrategyWithSingleStage(nil, nil) // Set smaller wait time for testing stageUpdatingWaitTime = time.Second * 3 @@ -745,7 +870,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -833,7 +958,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Validating the 3rd cluster has succeeded and stage waiting for AfterStageTasks") wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") @@ -851,7 +976,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -944,7 +1069,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) wantStatus.StagesStatus[0].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") @@ -983,7 +1108,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1079,7 +1204,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1129,7 +1254,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") By("Validating the updateRun is stuck in the 1st cluster of the 1st stage") - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) wantStatus.Conditions[1].Reason = condition.UpdateRunStuckReason validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, wantStatus, "") @@ -1142,8 +1267,14 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { }) Context("Cluster staged update run should recreate deleted approvalRequest", Ordered, func() { + var wantApprovalRequest *placementv1beta1.ClusterApprovalRequest BeforeAll(func() { By("Creating a strategy with single stage and both after stage tasks") + updateStrategy.Spec.Stages[0].BeforeStageTasks = []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + } updateStrategy.Spec.Stages[0].AfterStageTasks = []placementv1beta1.StageTask{ { Type: placementv1beta1.StageTaskTypeApproval, @@ -1162,10 +1293,91 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Creating a new clusterStagedUpdateRun") Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) - By("Validating the initialization succeeded and the execution started") + By("Validating the initialization succeeded and the execution has not started") initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) - wantStatus = generateExecutionStartedStatus(updateRun, initialized) + wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the approvalRequest has been created") + wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[0].BeforeStageTaskStatus[0].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[0].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + + }) + + It("Should not start rolling out", func() { + By("Validating the 1st clusterResourceBinding is not updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBoundBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Validating the 1st stage does not have startTime set") + Expect(updateRun.Status.StagesStatus[0].StartTime).Should(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) + }) + + It("Should start the 1st stage after approval request is approved", func() { + By("Validating the approvalRequest has been created") + approvalRequest := &placementv1beta1.ClusterApprovalRequest{} + wantApprovalRequest := &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[0].BeforeStageTaskStatus[0].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[0].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + + By("Deleting the approvalRequest") + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, approvalRequest)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, approvalRequest)).Should(Succeed()) + + By("Validating the approvalRequest has been recreated immediately") + validateApprovalRequestCreated(wantApprovalRequest) + + By("Approving the approvalRequest") + approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) + + By("Check the updateRun status") + wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) + wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + wantStatus.StagesStatus[0].Conditions[0] = generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with progressing reason. + meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Deleting the approvalRequest") + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, approvalRequest)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, approvalRequest)).Should(Succeed(), "failed to delete the approvalRequest") + + By("Validating the approvalRequest has not been recreated") + Eventually(func() bool { + return apierrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, approvalRequest)) + }, timeout, interval).Should(BeTrue(), "failed to ensure the approvalRequest is not recreated") + Consistently(func() bool { + return apierrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, approvalRequest)) + }, timeout, interval).Should(BeTrue(), "failed to ensure the approvalRequest is not recreated") }) It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -1215,7 +1427,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.StagesStatus[0].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) @@ -1277,7 +1489,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) // Need to have a longer wait time for the test to pass, because of the long wait time specified in the update strategy. timeout = time.Second * 90 @@ -1336,6 +1548,23 @@ func validateBindingState(ctx context.Context, binding *placementv1beta1.Cluster }, timeout, interval).Should(Succeed(), "failed to validate the binding state") } +func validateNotBoundBindingState(ctx context.Context, binding *placementv1beta1.ClusterResourceBinding, resourceSnapshotName string, updateRun *placementv1beta1.ClusterStagedUpdateRun, stage int) { + Consistently(func() error { + if err := k8sClient.Get(ctx, types.NamespacedName{Name: binding.Name}, binding); err != nil { + return err + } + + if binding.Spec.State == placementv1beta1.BindingStateBound { + return fmt.Errorf("binding %s is in Bound state, got %s", binding.Name, binding.Spec.State) + } + rolloutStartedCond := binding.GetCondition(string(placementv1beta1.ResourceBindingRolloutStarted)) + if condition.IsConditionStatusTrue(rolloutStartedCond, binding.Generation) { + return fmt.Errorf("binding %s rollout has started", binding.Name) + } + return nil + }, duration, interval).Should(Succeed(), "failed to validate the binding state") +} + func approveClusterApprovalRequest(ctx context.Context, approvalRequestName string) { Eventually(func() error { var approvalRequest placementv1beta1.ClusterApprovalRequest diff --git a/pkg/controllers/updaterun/execution_test.go b/pkg/controllers/updaterun/execution_test.go index ce775ea50..6d8650d60 100644 --- a/pkg/controllers/updaterun/execution_test.go +++ b/pkg/controllers/updaterun/execution_test.go @@ -19,6 +19,7 @@ package updaterun import ( "context" "errors" + "fmt" "strings" "testing" "time" @@ -943,3 +944,256 @@ func TestCalculateMaxConcurrencyValue(t *testing.T) { }) } } + +func TestCheckBeforeStageTasksStatus_NegativeCases(t *testing.T) { + stageName := "stage-0" + testUpdateRunName = "test-update-run" + approvalRequestName := fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName) + tests := []struct { + name string + stageIndex int + updateRun *placementv1beta1.ClusterStagedUpdateRun + approvalRequest *placementv1beta1.ClusterApprovalRequest + wantErrMsg string + wantErrAborted bool + }{ + // Negative test cases only + { + name: "should return err if before stage task is TimedWait", + stageIndex: 0, + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: stageName, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeTimedWait, + }, + }, + }, + }, + }, + StagesStatus: []placementv1beta1.StageUpdatingStatus{ + { + StageName: stageName, + BeforeStageTaskStatus: []placementv1beta1.StageTaskStatus{ + { + Type: placementv1beta1.StageTaskTypeTimedWait, + }, + }, + }, + }, + }, + }, + wantErrMsg: fmt.Sprintf("found unsupported task type in before stage tasks: %s", placementv1beta1.StageTaskTypeTimedWait), + wantErrAborted: true, + }, + { + name: "should return err if Approval request has wrong target stage in spec", + stageIndex: 0, + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: testUpdateRunName, + }, + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: stageName, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + }, + }, + }, + StagesStatus: []placementv1beta1.StageUpdatingStatus{ + { + StageName: stageName, + BeforeStageTaskStatus: []placementv1beta1.StageTaskStatus{ + { + Type: placementv1beta1.StageTaskTypeApproval, + ApprovalRequestName: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName), + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.StageTaskConditionApprovalRequestCreated), + Status: metav1.ConditionTrue, + }, + }, + }, + }, + }, + }, + }, + }, + approvalRequest: &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: approvalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: testUpdateRunName, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: testUpdateRunName, + TargetStage: "stage-1", + }, + }, + wantErrMsg: fmt.Sprintf("the approval request task `/%s` is targeting update run `/%s` and stage `stage-1`", approvalRequestName, testUpdateRunName), + wantErrAborted: true, + }, + { + name: "should return err if Approval request has wrong target update run in spec", + stageIndex: 0, + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: testUpdateRunName, + }, + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: stageName, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + }, + }, + }, + StagesStatus: []placementv1beta1.StageUpdatingStatus{ + { + StageName: stageName, + BeforeStageTaskStatus: []placementv1beta1.StageTaskStatus{ + { + Type: placementv1beta1.StageTaskTypeApproval, + ApprovalRequestName: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName), + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.StageTaskConditionApprovalRequestCreated), + Status: metav1.ConditionTrue, + }, + }, + }, + }, + }, + }, + }, + }, + approvalRequest: &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName), + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: testUpdateRunName, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "wrong-update-run", + TargetStage: stageName, + }, + }, + wantErrMsg: fmt.Sprintf("the approval request task `/%s` is targeting update run `/wrong-update-run` and stage `%s`", approvalRequestName, stageName), + wantErrAborted: true, + }, + { + name: "should return err if cannot update Approval request that is approved as accepted", + stageIndex: 0, + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: testUpdateRunName, + }, + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: stageName, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + }, + }, + }, + StagesStatus: []placementv1beta1.StageUpdatingStatus{ + { + StageName: stageName, + BeforeStageTaskStatus: []placementv1beta1.StageTaskStatus{ + { + Type: placementv1beta1.StageTaskTypeApproval, + ApprovalRequestName: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName), + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.StageTaskConditionApprovalRequestCreated), + Status: metav1.ConditionTrue, + }, + }, + }, + }, + }, + }, + }, + }, + approvalRequest: &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName), + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: testUpdateRunName, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: testUpdateRunName, + TargetStage: stageName, + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + Status: metav1.ConditionTrue, + }, + }, + }, + }, + wantErrMsg: fmt.Sprintf("error returned by the API server: clusterapprovalrequests.placement.kubernetes-fleet.io \"%s\" not found", approvalRequestName), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objects := []client.Object{tt.updateRun} + if tt.approvalRequest != nil { + objects = append(objects, tt.approvalRequest) + } + objectsWithStatus := []client.Object{tt.updateRun} + scheme := runtime.NewScheme() + _ = placementv1beta1.AddToScheme(scheme) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource(objectsWithStatus...). + Build() + r := Reconciler{ + Client: fakeClient, + } + ctx := context.Background() + _, gotErr := r.checkBeforeStageTasksStatus(ctx, tt.stageIndex, tt.updateRun) + if gotErr == nil { + t.Fatalf("checkBeforeStageTasksStatus() want error but got nil") + } + if !strings.Contains(gotErr.Error(), tt.wantErrMsg) { + t.Fatalf("checkBeforeStageTasksStatus() error = %v, wantErr %v", gotErr, tt.wantErrMsg) + } + if tt.wantErrAborted && !errors.Is(gotErr, errStagedUpdatedAborted) { + t.Fatalf("checkBeforeStageTasksStatus() want aborted error but got different error: %v", gotErr) + } + }) + } +} diff --git a/pkg/controllers/updaterun/initialization.go b/pkg/controllers/updaterun/initialization.go index 1296008c2..baa07be2b 100644 --- a/pkg/controllers/updaterun/initialization.go +++ b/pkg/controllers/updaterun/initialization.go @@ -292,7 +292,7 @@ func (r *Reconciler) generateStagesByStrategy( updateStrategySpec := updateStrategy.GetUpdateStrategySpec() updateRunStatus.UpdateStrategySnapshot = updateStrategySpec - // Remove waitTime from the updateRun status for AfterStageTask for type Approval. + // Remove waitTime from the updateRun status for BeforeStageTask and AfterStageTask for type Approval. removeWaitTimeFromUpdateRunStatus(updateRun) // Compute the update stages. @@ -344,6 +344,12 @@ func (r *Reconciler) computeRunStageStatus( // Apply the label selectors from the UpdateStrategy to filter the clusters. for _, stage := range updateRunStatus.UpdateStrategySnapshot.Stages { + if err := validateBeforeStageTask(stage.BeforeStageTasks); err != nil { + klog.ErrorS(err, "Failed to validate the before stage tasks", "updateStrategy", strategyKey, "stageName", stage.Name, "updateRun", updateRunRef) + // no more retries here. + invalidBeforeStageErr := controller.NewUserError(fmt.Errorf("the before stage tasks are invalid, updateStrategy: `%s`, stage: %s, err: %s", strategyKey, stage.Name, err.Error())) + return fmt.Errorf("%w: %s", errInitializedFailed, invalidBeforeStageErr.Error()) + } if err := validateAfterStageTask(stage.AfterStageTasks); err != nil { klog.ErrorS(err, "Failed to validate the after stage tasks", "updateStrategy", strategyKey, "stageName", stage.Name, "updateRun", updateRunRef) // no more retries here. @@ -418,12 +424,20 @@ func (r *Reconciler) computeRunStageStatus( curStageUpdatingStatus.Clusters[i].ClusterName = cluster.Name } + // Create the before stage tasks. + curStageUpdatingStatus.BeforeStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.BeforeStageTasks)) + for i, task := range stage.BeforeStageTasks { + curStageUpdatingStatus.BeforeStageTaskStatus[i].Type = task.Type + if task.Type == placementv1beta1.StageTaskTypeApproval { + curStageUpdatingStatus.BeforeStageTaskStatus[i].ApprovalRequestName = fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + } // Create the after stage tasks. curStageUpdatingStatus.AfterStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.AfterStageTasks)) for i, task := range stage.AfterStageTasks { curStageUpdatingStatus.AfterStageTaskStatus[i].Type = task.Type if task.Type == placementv1beta1.StageTaskTypeApproval { - curStageUpdatingStatus.AfterStageTaskStatus[i].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + curStageUpdatingStatus.AfterStageTaskStatus[i].ApprovalRequestName = fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) } } stagesStatus = append(stagesStatus, curStageUpdatingStatus) @@ -448,8 +462,25 @@ func (r *Reconciler) computeRunStageStatus( return nil } -// validateAfterStageTask valides the afterStageTasks in the stage defined in the UpdateStrategy. -// The error returned from this function is not retryable. +// validateBeforeStageTask validates the beforeStageTasks in the stage defined in the UpdateStrategy. +// The error returned from this function is not retriable. +func validateBeforeStageTask(tasks []placementv1beta1.StageTask) error { + if len(tasks) > 1 { + return fmt.Errorf("beforeStageTasks can have at most one task") + } + for i, task := range tasks { + if task.Type != placementv1beta1.StageTaskTypeApproval { + return fmt.Errorf("task %d of type %s is not allowed in beforeStageTasks, allowed type: Approval", i, task.Type) + } + if task.WaitTime != nil { + return fmt.Errorf("task %d of type Approval cannot have wait duration set", i) + } + } + return nil +} + +// validateAfterStageTask validates the afterStageTasks in the stage defined in the UpdateStrategy. +// The error returned from this function is not retriable. func validateAfterStageTask(tasks []placementv1beta1.StageTask) error { if len(tasks) == 2 && tasks[0].Type == tasks[1].Type { return fmt.Errorf("afterStageTasks cannot have two tasks of the same type: %s", tasks[0].Type) diff --git a/pkg/controllers/updaterun/initialization_integration_test.go b/pkg/controllers/updaterun/initialization_integration_test.go index 9a4507eee..0351b8a77 100644 --- a/pkg/controllers/updaterun/initialization_integration_test.go +++ b/pkg/controllers/updaterun/initialization_integration_test.go @@ -874,14 +874,14 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the clusterStagedUpdateRun stats") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - want := generateExecutionStartedStatus(updateRun, initialized) + want := generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") By("Validating the clusterStagedUpdateRun initialized consistently") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, want, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) }) It("Should put related ClusterResourceOverrides in the status", func() { @@ -896,14 +896,14 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the clusterStagedUpdateRun stats") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - want := generateExecutionStartedStatus(updateRun, initialized) + want := generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") By("Validating the clusterStagedUpdateRun initialized consistently") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, want, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) }) It("Should pick latest master resource snapshot if multiple snapshots", func() { @@ -931,14 +931,14 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the clusterStagedUpdateRun status") initialized := generateSucceededInitializationStatus(crp, updateRun, "2", policySnapshot, updateStrategy, clusterResourceOverride) - want := generateExecutionStartedStatus(updateRun, initialized) + want := generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") By("Validating the clusterStagedUpdateRun initialized consistently") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, want, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) }) }) }) @@ -1010,15 +1010,25 @@ func generateSucceededInitializationStatus( }, } for i := range status.StagesStatus { - var tasks []placementv1beta1.StageTaskStatus + var beforeTasks []placementv1beta1.StageTaskStatus + for _, task := range updateStrategy.Spec.Stages[i].BeforeStageTasks { + taskStatus := placementv1beta1.StageTaskStatus{Type: task.Type} + if task.Type == placementv1beta1.StageTaskTypeApproval { + taskStatus.ApprovalRequestName = fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, updateRun.Name, status.StagesStatus[i].StageName) + } + beforeTasks = append(beforeTasks, taskStatus) + } + status.StagesStatus[i].BeforeStageTaskStatus = beforeTasks + + var afterTasks []placementv1beta1.StageTaskStatus for _, task := range updateStrategy.Spec.Stages[i].AfterStageTasks { taskStatus := placementv1beta1.StageTaskStatus{Type: task.Type} if task.Type == placementv1beta1.StageTaskTypeApproval { - taskStatus.ApprovalRequestName = updateRun.Name + "-" + status.StagesStatus[i].StageName + taskStatus.ApprovalRequestName = fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, updateRun.Name, status.StagesStatus[i].StageName) } - tasks = append(tasks, taskStatus) + afterTasks = append(afterTasks, taskStatus) } - status.StagesStatus[i].AfterStageTaskStatus = tasks + status.StagesStatus[i].AfterStageTaskStatus = afterTasks } return status } @@ -1056,28 +1066,56 @@ func generateSucceededInitializationStatusForSmallClusters( }, } for i := range status.StagesStatus { - var tasks []placementv1beta1.StageTaskStatus + var beforeTasks []placementv1beta1.StageTaskStatus + for _, task := range updateStrategy.Spec.Stages[i].BeforeStageTasks { + taskStatus := placementv1beta1.StageTaskStatus{Type: task.Type} + if task.Type == placementv1beta1.StageTaskTypeApproval { + taskStatus.ApprovalRequestName = fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, updateRun.Name, status.StagesStatus[i].StageName) + } + beforeTasks = append(beforeTasks, taskStatus) + } + status.StagesStatus[i].BeforeStageTaskStatus = beforeTasks + + var afterTasks []placementv1beta1.StageTaskStatus for _, task := range updateStrategy.Spec.Stages[i].AfterStageTasks { taskStatus := placementv1beta1.StageTaskStatus{Type: task.Type} if task.Type == placementv1beta1.StageTaskTypeApproval { - taskStatus.ApprovalRequestName = updateRun.Name + "-" + status.StagesStatus[i].StageName + taskStatus.ApprovalRequestName = fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, updateRun.Name, status.StagesStatus[i].StageName) } - tasks = append(tasks, taskStatus) + afterTasks = append(afterTasks, taskStatus) } - status.StagesStatus[i].AfterStageTaskStatus = tasks + status.StagesStatus[i].AfterStageTaskStatus = afterTasks } return status } func generateExecutionStartedStatus( updateRun *placementv1beta1.ClusterStagedUpdateRun, - initialized *placementv1beta1.UpdateRunStatus, + status *placementv1beta1.UpdateRunStatus, ) *placementv1beta1.UpdateRunStatus { // Mark updateRun execution has started. - initialized.Conditions = append(initialized.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + meta.SetStatusCondition(&status.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + // Mark updateRun 1st stage has started. - initialized.StagesStatus[0].Conditions = append(initialized.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + meta.SetStatusCondition(&status.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + // Mark updateRun 1st cluster in the 1st stage has started. - initialized.StagesStatus[0].Clusters[0].Conditions = []metav1.Condition{generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)} - return initialized + status.StagesStatus[0].Clusters[0].Conditions = []metav1.Condition{generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)} + return status +} + +func generateExecutionNotStartedStatus( + updateRun *placementv1beta1.ClusterStagedUpdateRun, + status *placementv1beta1.UpdateRunStatus, +) *placementv1beta1.UpdateRunStatus { + // Mark updateRun execution has not started. + status.Conditions = append(status.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + + // Mark updateRun 1st stage has not started. + status.StagesStatus[0].Conditions = append(status.StagesStatus[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + + // Mark updateRun 1st stage BeforeStageTasks has created approval request. + status.StagesStatus[0].BeforeStageTaskStatus[0].Conditions = append(status.StagesStatus[0].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) + return status } diff --git a/pkg/controllers/updaterun/initialization_test.go b/pkg/controllers/updaterun/initialization_test.go index 0c470d38e..6ce3689a8 100644 --- a/pkg/controllers/updaterun/initialization_test.go +++ b/pkg/controllers/updaterun/initialization_test.go @@ -1,30 +1,114 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package updaterun import ( + "fmt" "testing" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" - "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" + placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" ) +func TestValidateBeforeStageTask(t *testing.T) { + tests := []struct { + name string + task []placementv1beta1.StageTask + wantErr bool + wantErrMsg string + }{ + { + name: "valid BeforeTasks", + task: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + wantErr: false, + }, + { + name: "invalid BeforeTasks, greater than 1 task", + task: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + wantErr: true, + wantErrMsg: "beforeStageTasks can have at most one task", + }, + { + name: "invalid BeforeTasks, with invalid task type", + task: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeTimedWait, + WaitTime: ptr.To(metav1.Duration{Duration: 5 * time.Minute}), + }, + }, + wantErr: true, + wantErrMsg: fmt.Sprintf("task %d of type %s is not allowed in beforeStageTasks, allowed type: Approval", 0, placementv1beta1.StageTaskTypeTimedWait), + }, + { + name: "invalid BeforeTasks, with duration for Approval", + task: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + WaitTime: ptr.To(metav1.Duration{Duration: 1 * time.Minute}), + }, + }, + wantErr: true, + wantErrMsg: fmt.Sprintf("task %d of type Approval cannot have wait duration set", 0), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotErr := validateBeforeStageTask(tt.task) + if tt.wantErr { + if gotErr == nil || gotErr.Error() != tt.wantErrMsg { + t.Fatalf("validateBeforeStageTask() error = %v, wantErr %v", gotErr, tt.wantErrMsg) + } + } else if gotErr != nil { + t.Fatalf("validateBeforeStageTask() error = %v, wantErr %v", gotErr, tt.wantErr) + } + }) + } +} + func TestValidateAfterStageTask(t *testing.T) { tests := []struct { name string - task []v1beta1.StageTask + task []placementv1beta1.StageTask wantErr bool errMsg string }{ { name: "valid AfterTasks", - task: []v1beta1.StageTask{ + task: []placementv1beta1.StageTask{ { - Type: v1beta1.StageTaskTypeApproval, + Type: placementv1beta1.StageTaskTypeApproval, }, { - Type: v1beta1.StageTaskTypeTimedWait, + Type: placementv1beta1.StageTaskTypeTimedWait, WaitTime: ptr.To(metav1.Duration{Duration: 5 * time.Minute}), }, }, @@ -32,13 +116,13 @@ func TestValidateAfterStageTask(t *testing.T) { }, { name: "invalid AfterTasks, same type of tasks", - task: []v1beta1.StageTask{ + task: []placementv1beta1.StageTask{ { - Type: v1beta1.StageTaskTypeTimedWait, + Type: placementv1beta1.StageTaskTypeTimedWait, WaitTime: ptr.To(metav1.Duration{Duration: 1 * time.Minute}), }, { - Type: v1beta1.StageTaskTypeTimedWait, + Type: placementv1beta1.StageTaskTypeTimedWait, WaitTime: ptr.To(metav1.Duration{Duration: 5 * time.Minute}), }, }, @@ -47,9 +131,9 @@ func TestValidateAfterStageTask(t *testing.T) { }, { name: "invalid AfterTasks, with nil duration for TimedWait", - task: []v1beta1.StageTask{ + task: []placementv1beta1.StageTask{ { - Type: v1beta1.StageTaskTypeTimedWait, + Type: placementv1beta1.StageTaskTypeTimedWait, }, }, wantErr: true, @@ -57,9 +141,9 @@ func TestValidateAfterStageTask(t *testing.T) { }, { name: "invalid AfterTasks, with zero duration for TimedWait", - task: []v1beta1.StageTask{ + task: []placementv1beta1.StageTask{ { - Type: v1beta1.StageTaskTypeTimedWait, + Type: placementv1beta1.StageTaskTypeTimedWait, WaitTime: ptr.To(metav1.Duration{Duration: 0 * time.Minute}), }, }, diff --git a/pkg/controllers/updaterun/validation_integration_test.go b/pkg/controllers/updaterun/validation_integration_test.go index bcc255473..b1190601e 100644 --- a/pkg/controllers/updaterun/validation_integration_test.go +++ b/pkg/controllers/updaterun/validation_integration_test.go @@ -111,7 +111,7 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the initialization succeeded") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - wantStatus = generateExecutionStartedStatus(updateRun, initialized) + wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) @@ -164,7 +164,7 @@ var _ = Describe("UpdateRun validation tests", func() { Context("Test validateCRP", func() { AfterEach(func() { By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateFailedMetric(updateRun)) }) It("Should fail to validate if the CRP is not found", func() { @@ -208,7 +208,7 @@ var _ = Describe("UpdateRun validation tests", func() { validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "no latest policy snapshot associated") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateFailedMetric(updateRun)) }) It("Should fail to validate if the latest policySnapshot has changed", func() { @@ -237,7 +237,7 @@ var _ = Describe("UpdateRun validation tests", func() { Expect(k8sClient.Delete(ctx, newPolicySnapshot)).Should(Succeed()) By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateFailedMetric(updateRun)) }) It("Should fail to validate if the cluster count has changed", func() { @@ -251,14 +251,14 @@ var _ = Describe("UpdateRun validation tests", func() { "the cluster count initialized in the updateRun is outdated") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateFailedMetric(updateRun)) }) }) Context("Test validateStagesStatus", func() { AfterEach(func() { By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateFailedMetric(updateRun)) }) It("Should fail to validate if the UpdateStrategySnapshot is nil", func() { @@ -443,7 +443,7 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the initialization succeeded") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - wantStatus = generateExecutionStartedStatus(updateRun, initialized) + wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) @@ -507,7 +507,7 @@ var _ = Describe("UpdateRun validation tests", func() { validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) }) }) }) diff --git a/pkg/utils/condition/reason.go b/pkg/utils/condition/reason.go index b1d963e58..9566ee42e 100644 --- a/pkg/utils/condition/reason.go +++ b/pkg/utils/condition/reason.go @@ -194,17 +194,20 @@ const ( // ClusterUpdatingSucceededReason is the reason string of condition if the cluster updating succeeded. ClusterUpdatingSucceededReason = "ClusterUpdatingSucceeded" - // AfterStageTaskApprovalRequestApprovedReason is the reason string of condition if the approval request for after stage task has been approved. - AfterStageTaskApprovalRequestApprovedReason = "AfterStageTaskApprovalRequestApproved" + // StageTaskApprovalRequestApprovedReason is the reason string of condition if the approval request for before or after stage task has been approved. + StageTaskApprovalRequestApprovedReason = "StageTaskApprovalRequestApproved" - // AfterStageTaskApprovalRequestCreatedReason is the reason string of condition if the approval request for after stage task has been created. - AfterStageTaskApprovalRequestCreatedReason = "AfterStageTaskApprovalRequestCreated" + // StageTaskApprovalRequestCreatedReason is the reason string of condition if the approval request for before or after stage task has been created. + StageTaskApprovalRequestCreatedReason = "StageTaskApprovalRequestCreated" // AfterStageTaskWaitTimeElapsedReason is the reason string of condition if the wait time for after stage task has elapsed. AfterStageTaskWaitTimeElapsedReason = "AfterStageTaskWaitTimeElapsed" // ApprovalRequestApprovalAcceptedReason is the reason string of condition if the approval of the approval request has been accepted. ApprovalRequestApprovalAcceptedReason = "ApprovalRequestApprovalAccepted" + + // UpdateRunWaitingMessageFmt is the message format string of condition if the staged update run is waiting for stage tasks in a stage to complete. + UpdateRunWaitingMessageFmt = "The updateRun is waiting for %s tasks in stage %s to complete" ) // A group of condition reason & message string which is used to populate the ClusterResourcePlacementEviction condition. diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index 344faf6f4..fc171ea48 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -2040,19 +2040,19 @@ func updateRunStageRolloutSucceedConditions(generation int64) []metav1.Condition } } -func updateRunAfterStageTaskSucceedConditions(generation int64, taskType placementv1beta1.StageTaskType) []metav1.Condition { +func updateRunStageTaskSucceedConditions(generation int64, taskType placementv1beta1.StageTaskType) []metav1.Condition { if taskType == placementv1beta1.StageTaskTypeApproval { return []metav1.Condition{ { Type: string(placementv1beta1.StageTaskConditionApprovalRequestCreated), Status: metav1.ConditionTrue, - Reason: condition.AfterStageTaskApprovalRequestCreatedReason, + Reason: condition.StageTaskApprovalRequestCreatedReason, ObservedGeneration: generation, }, { Type: string(placementv1beta1.StageTaskConditionApprovalRequestApproved), Status: metav1.ConditionTrue, - Reason: condition.AfterStageTaskApprovalRequestApprovedReason, + Reason: condition.StageTaskApprovalRequestApprovedReason, ObservedGeneration: generation, }, } @@ -2178,13 +2178,21 @@ func buildStageUpdatingStatuses( stagesStatus[i].Clusters[j].ResourceOverrideSnapshots = wantROs[wantSelectedClusters[i][j]] stagesStatus[i].Clusters[j].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.GetGeneration()) } + stagesStatus[i].BeforeStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.BeforeStageTasks)) + for j, task := range stage.BeforeStageTasks { + stagesStatus[i].BeforeStageTaskStatus[j].Type = task.Type + if task.Type == placementv1beta1.StageTaskTypeApproval { + stagesStatus[i].BeforeStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + stagesStatus[i].BeforeStageTaskStatus[j].Conditions = updateRunStageTaskSucceedConditions(updateRun.GetGeneration(), task.Type) + } stagesStatus[i].AfterStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.AfterStageTasks)) for j, task := range stage.AfterStageTasks { stagesStatus[i].AfterStageTaskStatus[j].Type = task.Type if task.Type == placementv1beta1.StageTaskTypeApproval { - stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) } - stagesStatus[i].AfterStageTaskStatus[j].Conditions = updateRunAfterStageTaskSucceedConditions(updateRun.GetGeneration(), task.Type) + stagesStatus[i].AfterStageTaskStatus[j].Conditions = updateRunStageTaskSucceedConditions(updateRun.GetGeneration(), task.Type) } stagesStatus[i].Conditions = updateRunStageRolloutSucceedConditions(updateRun.GetGeneration()) } diff --git a/test/e2e/cluster_staged_updaterun_test.go b/test/e2e/cluster_staged_updaterun_test.go index 8e8b28822..5bae5faa9 100644 --- a/test/e2e/cluster_staged_updaterun_test.go +++ b/test/e2e/cluster_staged_updaterun_test.go @@ -144,14 +144,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should rollout resources to member-cluster-1 first because of its name", func() { - checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { + It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -214,10 +216,20 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should not rollout resources to prod stage until approved", func() { + By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { + It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") @@ -312,14 +324,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should rollout resources to member-cluster-1 first because of its name", func() { - checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { + It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -381,10 +395,20 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { + It("Should not rollout resources to prod stage until approved", func() { + By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } + }) + + It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") @@ -419,10 +443,20 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{resourceSnapshotIndex2nd, resourceSnapshotIndex1st, resourceSnapshotIndex2nd}, []bool{true, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should not rollback resources to prod stage until approved", func() { + By("Verify that the configmap is not rolled back on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &newConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } }) - It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { + It("Should rollback resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) for idx := range allMemberClusters { @@ -515,10 +549,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[:2], []string{"", resourceSnapshotIndex1st}, []bool{false, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to member-cluster-1 too but not member-cluster-3 and complete the cluster staged update run successfully", func() { + It("Should rollout resources to member-cluster-1 after approval but not member-cluster-3 and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) @@ -565,10 +605,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should rollout resources to member-cluster-3 too and complete the cluster staged update run successfully", func() { + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) + }) + + It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -611,10 +657,12 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, false, []string{allMemberClusterNames[2]}, []string{resourceSnapshotIndex1st}, []bool{false}, nil, nil) Consistently(crpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should remove resources on member-cluster-1 and member-cluster-2 and complete the cluster staged update run successfully", func() { + It("Should remove resources on member-cluster-1 and member-cluster-2 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + // need to go through two stages csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) @@ -704,10 +752,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[2:], []string{""}, []bool{false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) }) - It("Should rollout resources to member-cluster-3 and complete the cluster staged update run successfully", func() { + It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[2]}) @@ -753,10 +807,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{false, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should not rollout resources to member-cluster-1 until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0]}) }) - It("Should rollout resources to member-cluster-1 too and complete the cluster staged update run successfully", func() { + It("Should rollout resources to member-cluster-1 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -799,10 +859,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames[1:], []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true}, nil, nil) Consistently(crpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should remove resources on member-cluster-1 and complete the cluster staged update run successfully", func() { + It("Should not remove resources from member-cluster-1 until approved", func() { + checkIfPlacedWorkResourcesOnMemberClustersConsistently(allMemberClusters) + }) + + It("Should remove resources on member-cluster-1 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0]}) @@ -971,10 +1037,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, wantROs) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunName, envCanary) + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should not rollout resources to member-cluster-1 and member-cluster-3 until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { + It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -1071,10 +1143,12 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunName, envCanary) + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should report diff for member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { + It("Should report diff for member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) }) @@ -1183,7 +1257,11 @@ var _ = Describe("test CRP rollout with staged update run", func() { configMapActual := configMapPlacedOnClusterActual(allMemberClusters[1], &newConfigMap) Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s", newConfigMap.Name, allMemberClusterNames[1]) - validateAndApproveClusterApprovalRequests(updateRunName, envCanary) + // Approval for AfterStageTasks of canary stage + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + + // Approval for BeforeStageTasks of prod stage + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) // Verify complete rollout csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) @@ -1265,50 +1343,12 @@ var _ = Describe("test CRP rollout with staged update run", func() { }, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to find cluster approval request") }) - It("Should approve cluster approval request using kubectl-fleet approve plugin", func() { - var approvalRequestName string - - // Get the cluster approval request name. - Eventually(func() error { - appReqList := &placementv1beta1.ClusterApprovalRequestList{} - if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ - placementv1beta1.TargetUpdatingStageNameLabel: envCanary, - placementv1beta1.TargetUpdateRunLabel: updateRunName, - }); err != nil { - return fmt.Errorf("failed to list approval requests: %w", err) - } - - if len(appReqList.Items) != 1 { - return fmt.Errorf("want 1 approval request, got %d", len(appReqList.Items)) - } - - approvalRequestName = appReqList.Items[0].Name - return nil - }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to get approval request name") - - // Use kubectl-fleet approve plugin to approve the request - cmd := exec.Command(fleetBinaryPath, "approve", "clusterapprovalrequest", - "--hubClusterContext", "kind-hub", - "--name", approvalRequestName) - output, err := cmd.CombinedOutput() - Expect(err).ToNot(HaveOccurred(), "kubectl-fleet approve failed: %s", string(output)) - - // Verify the approval request is approved - Eventually(func() error { - var appReq placementv1beta1.ClusterApprovalRequest - if err := hubClient.Get(ctx, client.ObjectKey{Name: approvalRequestName}, &appReq); err != nil { - return fmt.Errorf("failed to get approval request: %w", err) - } + It("Should approve after-stage cluster approval request using kubectl-fleet approve plugin for canary stage", func() { + approveClusterApprovalRequest(envCanary, updateRunName) + }) - approvedCondition := meta.FindStatusCondition(appReq.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)) - if approvedCondition == nil { - return fmt.Errorf("approved condition not found") - } - if approvedCondition.Status != metav1.ConditionTrue { - return fmt.Errorf("approved condition status is %s, want True", approvedCondition.Status) - } - return nil - }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to verify approval request is approved") + It("Should approve before-stage cluster approval request using kubectl-fleet approve plugin for prod stage", func() { + approveClusterApprovalRequest(envProd, updateRunName) }) It("Should complete the staged update run after approval", func() { @@ -1383,7 +1423,11 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Create updateRun and verify resources are rolled out", func() { createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) - validateAndApproveClusterApprovalRequests(updateRunName, envCanary) + // Approval for AfterStageTasks of canary stage + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + + // Approval for BeforeStageTasks of prod stage + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) @@ -1956,6 +2000,11 @@ func createClusterStagedUpdateStrategySucceed(strategyName string) *placementv1b envLabelName: envProd, // member-cluster-1 and member-cluster-3 }, }, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, }, }, }, @@ -2037,7 +2086,7 @@ func createClusterStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNam Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create ClusterStagedUpdateRun %s", updateRunName) } -func validateAndApproveClusterApprovalRequests(updateRunName, stageName string) { +func validateAndApproveClusterApprovalRequests(updateRunName, stageName, approvalRequestNameFmt string) { Eventually(func() error { appReqList := &placementv1beta1.ClusterApprovalRequestList{} if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ @@ -2051,6 +2100,10 @@ func validateAndApproveClusterApprovalRequests(updateRunName, stageName string) return fmt.Errorf("got %d approval requests, want 1", len(appReqList.Items)) } appReq := &appReqList.Items[0] + approvalRequestName := fmt.Sprintf(approvalRequestNameFmt, updateRunName, stageName) + if appReq.Name != approvalRequestName { + return fmt.Errorf("got approval request %s, want %s", appReq.Name, approvalRequestName) + } meta.SetStatusCondition(&appReq.Status.Conditions, metav1.Condition{ Status: metav1.ConditionTrue, Type: string(placementv1beta1.ApprovalRequestConditionApproved), @@ -2068,3 +2121,49 @@ func updateConfigMapSucceed(newConfigMap *corev1.ConfigMap) { cm.Data = newConfigMap.Data Expect(hubClient.Update(ctx, cm)).To(Succeed(), "Failed to update configmap %s in namespace %s", newConfigMap.Name, newConfigMap.Namespace) } + +func approveClusterApprovalRequest(stageName, updateRunName string) { + var approvalRequestName string + + // Get the cluster approval request name. + Eventually(func() error { + appReqList := &placementv1beta1.ClusterApprovalRequestList{} + if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: updateRunName, + }); err != nil { + return fmt.Errorf("failed to list approval requests: %w", err) + } + + if len(appReqList.Items) != 1 { + return fmt.Errorf("want 1 approval request, got %d", len(appReqList.Items)) + } + + approvalRequestName = appReqList.Items[0].Name + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to get approval request name") + + // Use kubectl-fleet approve plugin to approve the request + cmd := exec.Command(fleetBinaryPath, "approve", "clusterapprovalrequest", + "--hubClusterContext", "kind-hub", + "--name", approvalRequestName) + output, err := cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred(), "kubectl-fleet approve failed: %s", string(output)) + + // Verify the approval request is approved + Eventually(func() error { + var appReq placementv1beta1.ClusterApprovalRequest + if err := hubClient.Get(ctx, client.ObjectKey{Name: approvalRequestName}, &appReq); err != nil { + return fmt.Errorf("failed to get approval request: %w", err) + } + + approvedCondition := meta.FindStatusCondition(appReq.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)) + if approvedCondition == nil { + return fmt.Errorf("approved condition not found") + } + if approvedCondition.Status != metav1.ConditionTrue { + return fmt.Errorf("approved condition status is %s, want True", approvedCondition.Status) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to verify approval request is approved") +} diff --git a/test/e2e/staged_updaterun_test.go b/test/e2e/staged_updaterun_test.go index 2f03affa9..60dcfe885 100644 --- a/test/e2e/staged_updaterun_test.go +++ b/test/e2e/staged_updaterun_test.go @@ -135,14 +135,16 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should rollout resources to member-cluster-1 first because of its name", func() { - checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to all the members and complete the staged update run successfully", func() { + It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -204,10 +206,19 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + It("Should not rollout resources to prod stage until approved", func() { + By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") @@ -300,14 +311,16 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should rollout resources to member-cluster-1 first because of its name", func() { - checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to all the members and complete the staged update run successfully", func() { + It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -369,10 +382,20 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should not rollout resources to prod stage until approved", func() { + By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") @@ -407,10 +430,20 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{resourceSnapshotIndex2nd, resourceSnapshotIndex1st, resourceSnapshotIndex2nd}, []bool{true, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + It("Should not rollback resources to prod stage until approved", func() { + By("Verify that the configmap is not rolled back on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &newConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } + }) + + It("Should rollback resources to member-cluster-1 and member-cluster-3 after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) for idx := range allMemberClusters { @@ -501,10 +534,16 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[:2], []string{"", resourceSnapshotIndex1st}, []bool{false, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to member-cluster-1 too but not member-cluster-3 and complete the staged update run successfully", func() { + It("Should rollout resources to member-cluster-1 after approval but not member-cluster-3 and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) @@ -551,10 +590,16 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) }) - It("Should rollout resources to member-cluster-3 too and complete the staged update run successfully", func() { + It("Should rollout resources to member-cluster-3 after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -597,10 +642,12 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, false, []string{allMemberClusterNames[2]}, []string{resourceSnapshotIndex1st}, []bool{false}, nil, nil) Consistently(rpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should remove resources on member-cluster-1 and member-cluster-2 and complete the staged update run successfully", func() { + It("Should remove resources on member-cluster-1 and member-cluster-2 after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + // need to go through two stages surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) Eventually(surSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[2]) @@ -688,10 +735,16 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[2:], []string{""}, []bool{false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should rollout resources to member-cluster-3 and complete the staged update run successfully", func() { + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) + }) + + It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[2]}) @@ -737,10 +790,16 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{false, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should rollout resources to member-cluster-1 too and complete the staged update run successfully", func() { + It("Should not rollout resources to member-cluster-1 until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0]}) + }) + + It("Should rollout resources to member-cluster-1 after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -783,10 +842,16 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames[1:], []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true}, nil, nil) Consistently(rpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should not remove resources from member-cluster-1 until approved", func() { + checkIfPlacedWorkResourcesOnMemberClustersConsistently(allMemberClusters) }) - It("Should remove resources on member-cluster-1 and complete the staged update run successfully", func() { + It("Should remove resources on member-cluster-1 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) Eventually(surSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[2]) checkIfRemovedConfigMapFromMemberClusters([]*framework.Cluster{allMemberClusters[0]}) @@ -928,10 +993,16 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, wantROs) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should not rollout resources to member-cluster-1 and member-cluster-3 until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, wantROs) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -1022,10 +1093,12 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) }) - It("Should report diff for member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + It("Should report diff for member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) }) @@ -1132,7 +1205,11 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem configMapActual := configMapPlacedOnClusterActual(allMemberClusters[1], &newConfigMap) Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s", newConfigMap.Name, allMemberClusterNames[1]) - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) + // Approval for AfterStageTask of canary stage + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + + // Approval for BeforeStageTask of prod stage + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) // Verify complete rollout. surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) @@ -1209,7 +1286,11 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Create updateRun and verify resources are rolled out", func() { createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) + // Approval for AfterStageTask of canary stage + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + + // Approval for BeforeStageTask of prod stage + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) @@ -1481,6 +1562,11 @@ func createStagedUpdateStrategySucceed(strategyName, namespace string) *placemen envLabelName: envProd, // member-cluster-1 and member-cluster-3 }, }, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, }, }, }, @@ -1564,7 +1650,7 @@ func createStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunName, name Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create StagedUpdateRun %s", updateRunName) } -func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stageName string) { +func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stageName, approvalRequestNameFmt string) { Eventually(func() error { appReqList := &placementv1beta1.ApprovalRequestList{} if err := hubClient.List(ctx, appReqList, client.InNamespace(namespace), client.MatchingLabels{ @@ -1578,6 +1664,10 @@ func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stag return fmt.Errorf("got %d approval requests, want 1", len(appReqList.Items)) } appReq := &appReqList.Items[0] + approvalRequestName := fmt.Sprintf(approvalRequestNameFmt, updateRunName, stageName) + if appReq.Name != approvalRequestName { + return fmt.Errorf("got approval request %s, want %s", appReq.Name, approvalRequestName) + } meta.SetStatusCondition(&appReq.Status.Conditions, metav1.Condition{ Status: metav1.ConditionTrue, Type: string(placementv1beta1.ApprovalRequestConditionApproved), From 2b1e9a71de5f9bbe7f944c4a9c64991ae88b0f43 Mon Sep 17 00:00:00 2001 From: Britania Rodriguez Reyes <145056127+britaniar@users.noreply.github.com> Date: Wed, 3 Dec 2025 14:47:36 -0600 Subject: [PATCH 06/16] feat: implement initialize and execute states within update run (#346) --- apis/placement/v1beta1/stageupdate_types.go | 10 +- pkg/controllers/updaterun/controller.go | 60 +++-- .../updaterun/controller_integration_test.go | 28 +-- pkg/controllers/updaterun/execution.go | 54 +++-- .../updaterun/execution_integration_test.go | 169 +++++++++++--- pkg/controllers/updaterun/initialization.go | 2 +- .../updaterun/validation_integration_test.go | 3 +- .../api_validation_integration_test.go | 39 ++-- test/e2e/actuals_test.go | 87 ++++++- test/e2e/cluster_staged_updaterun_test.go | 213 ++++++++++++++---- test/e2e/staged_updaterun_test.go | 186 ++++++++++++--- 11 files changed, 649 insertions(+), 202 deletions(-) diff --git a/apis/placement/v1beta1/stageupdate_types.go b/apis/placement/v1beta1/stageupdate_types.go index 588e38136..bc7a8d212 100644 --- a/apis/placement/v1beta1/stageupdate_types.go +++ b/apis/placement/v1beta1/stageupdate_types.go @@ -152,13 +152,14 @@ func (c *ClusterStagedUpdateRun) SetUpdateRunStatus(status UpdateRunStatus) { type State string const ( - // StateNotStarted describes user intent to initialize but not execute the update run. + // StateInitialized describes user intent to initialize but not execute the update run. // This is the default state when an update run is created. - StateNotStarted State = "Initialize" + // Users can subsequently set the state to Execute or Abandon. + StateInitialized State = "Initialize" - // StateStarted describes user intent to execute (or resume execution if paused). + // StateExecuted describes user intent to execute (or resume execution if paused). // Users can subsequently set the state to Pause or Abandon. - StateStarted State = "Execute" + StateExecuted State = "Execute" // StateStopped describes user intent to pause the update run. // Users can subsequently set the state to Execute or Abandon. @@ -426,7 +427,6 @@ const ( // Its condition status can be one of the following: // - "True": The staged update run is initialized successfully. // - "False": The staged update run encountered an error during initialization and aborted. - // - "Unknown": The staged update run initialization has started. StagedUpdateRunConditionInitialized StagedUpdateRunConditionType = "Initialized" // StagedUpdateRunConditionProgressing indicates whether the staged update run is making progress. diff --git a/pkg/controllers/updaterun/controller.go b/pkg/controllers/updaterun/controller.go index 4e8fa695c..efd1ced2c 100644 --- a/pkg/controllers/updaterun/controller.go +++ b/pkg/controllers/updaterun/controller.go @@ -104,15 +104,23 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim // Emit the update run status metric based on status conditions in the updateRun. defer emitUpdateRunStatusMetric(updateRun) + state := updateRun.GetUpdateRunSpec().State + var updatingStageIndex int var toBeUpdatedBindings, toBeDeletedBindings []placementv1beta1.BindingObj updateRunStatus := updateRun.GetUpdateRunStatus() initCond := meta.FindStatusCondition(updateRunStatus.Conditions, string(placementv1beta1.StagedUpdateRunConditionInitialized)) - if !condition.IsConditionStatusTrue(initCond, updateRun.GetGeneration()) { - if condition.IsConditionStatusFalse(initCond, updateRun.GetGeneration()) { + // Check if initialized regardless of generation. + // The updateRun spec fields are immutable except for the state field. When the state changes, + // the update run generation increments, but we don't need to reinitialize since initialization is a one-time setup. + if !(initCond != nil && initCond.Status == metav1.ConditionTrue) { + // Check if initialization failed for the current generation. + if initCond != nil && initCond.Status == metav1.ConditionFalse { klog.V(2).InfoS("The updateRun has failed to initialize", "errorMsg", initCond.Message, "updateRun", runObjRef) return runtime.Result{}, nil } + + // Initialize the updateRun. var initErr error if toBeUpdatedBindings, toBeDeletedBindings, initErr = r.initialize(ctx, updateRun); initErr != nil { klog.ErrorS(initErr, "Failed to initialize the updateRun", "updateRun", runObjRef) @@ -122,10 +130,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim } return runtime.Result{}, initErr } - updatingStageIndex = 0 // start from the first stage. - klog.V(2).InfoS("Initialized the updateRun", "updateRun", runObjRef) + updatingStageIndex = 0 // start from the first stage (typically for Initialize or Execute states). + klog.V(2).InfoS("Initialized the updateRun", "state", state, "updateRun", runObjRef) } else { - klog.V(2).InfoS("The updateRun is initialized", "updateRun", runObjRef) + klog.V(2).InfoS("The updateRun is initialized", "state", state, "updateRun", runObjRef) // Check if the updateRun is finished. finishedCond := meta.FindStatusCondition(updateRunStatus.Conditions, string(placementv1beta1.StagedUpdateRunConditionSucceeded)) if condition.IsConditionStatusTrue(finishedCond, updateRun.GetGeneration()) || condition.IsConditionStatusFalse(finishedCond, updateRun.GetGeneration()) { @@ -151,28 +159,32 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim } // Execute the updateRun. - klog.V(2).InfoS("Continue to execute the updateRun", "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) - finished, waitTime, execErr := r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings) - if errors.Is(execErr, errStagedUpdatedAborted) { - // errStagedUpdatedAborted cannot be retried. - return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, execErr.Error()) - } + if state == placementv1beta1.StateExecuted { + klog.V(2).InfoS("Continue to execute the updateRun", "state", state, "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) + finished, waitTime, execErr := r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings) + if errors.Is(execErr, errStagedUpdatedAborted) { + // errStagedUpdatedAborted cannot be retried. + return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, execErr.Error()) + } - if finished { - klog.V(2).InfoS("The updateRun is completed", "updateRun", runObjRef) - return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun) - } + if finished { + klog.V(2).InfoS("The updateRun is completed", "updateRun", runObjRef) + return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun) + } - // The execution is not finished yet or it encounters a retriable error. - // We need to record the status and requeue. - if updateErr := r.recordUpdateRunStatus(ctx, updateRun); updateErr != nil { - return runtime.Result{}, updateErr - } - klog.V(2).InfoS("The updateRun is not finished yet", "requeueWaitTime", waitTime, "execErr", execErr, "updateRun", runObjRef) - if execErr != nil { - return runtime.Result{}, execErr + // The execution is not finished yet or it encounters a retriable error. + // We need to record the status and requeue. + if updateErr := r.recordUpdateRunStatus(ctx, updateRun); updateErr != nil { + return runtime.Result{}, updateErr + } + klog.V(2).InfoS("The updateRun is not finished yet", "requeueWaitTime", waitTime, "execErr", execErr, "updateRun", runObjRef) + if execErr != nil { + return runtime.Result{}, execErr + } + return runtime.Result{Requeue: true, RequeueAfter: waitTime}, nil } - return runtime.Result{Requeue: true, RequeueAfter: waitTime}, nil + klog.V(2).InfoS("The updateRun is initialized but not executed, waiting to execute", "state", state, "updateRun", runObjRef) + return runtime.Result{}, nil } // handleDelete handles the deletion of the updateRun object. diff --git a/pkg/controllers/updaterun/controller_integration_test.go b/pkg/controllers/updaterun/controller_integration_test.go index d33b132b3..e017d170e 100644 --- a/pkg/controllers/updaterun/controller_integration_test.go +++ b/pkg/controllers/updaterun/controller_integration_test.go @@ -272,6 +272,16 @@ func generateMetricsLabels( } } +func generateInitializationSucceededMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionInitialized), + string(metav1.ConditionTrue), condition.UpdateRunInitializeSucceededReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + func generateInitializationFailedMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { return &prometheusclientmodel.Metric{ Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionInitialized), @@ -341,6 +351,7 @@ func generateTestClusterStagedUpdateRun() *placementv1beta1.ClusterStagedUpdateR PlacementName: testCRPName, ResourceSnapshotIndex: testResourceSnapshotIndex, StagedUpdateStrategyName: testUpdateStrategyName, + State: placementv1beta1.StateExecuted, }, } } @@ -807,23 +818,8 @@ func generateFalseCondition(obj client.Object, condType any) metav1.Condition { } } -func generateFalseProgressingCondition(obj client.Object, condType any, succeeded bool) metav1.Condition { +func generateFalseProgressingCondition(obj client.Object, condType any, reason string) metav1.Condition { falseCond := generateFalseCondition(obj, condType) - reason := "" - switch condType { - case placementv1beta1.StagedUpdateRunConditionProgressing: - if succeeded { - reason = condition.UpdateRunSucceededReason - } else { - reason = condition.UpdateRunFailedReason - } - case placementv1beta1.StageUpdatingConditionProgressing: - if succeeded { - reason = condition.StageUpdatingSucceededReason - } else { - reason = condition.StageUpdatingFailedReason - } - } falseCond.Reason = reason return falseCond } diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index 074c23739..1180ae34a 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -285,31 +285,45 @@ func (r *Reconciler) executeUpdatingStage( } if finishedClusterCount == len(updatingStageStatus.Clusters) { - // All the clusters in the stage have been updated. - markUpdateRunWaiting(updateRun, fmt.Sprintf(condition.UpdateRunWaitingMessageFmt, "after-stage", updatingStageStatus.StageName)) - markStageUpdatingWaiting(updatingStageStatus, updateRun.GetGeneration(), "All clusters in the stage are updated, waiting for after-stage tasks to complete") - klog.V(2).InfoS("The stage has finished all cluster updating", "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) - // Check if the after stage tasks are ready. - approved, waitTime, err := r.checkAfterStageTasksStatus(ctx, updatingStageIndex, updateRun) - if err != nil { - return 0, err - } - if approved { - markUpdateRunProgressing(updateRun) - markStageUpdatingSucceeded(updatingStageStatus, updateRun.GetGeneration()) - // No need to wait to get to the next stage. - return 0, nil - } - // The after stage tasks are not ready yet. - if waitTime < 0 { - waitTime = stageUpdatingWaitTime - } - return waitTime, nil + return r.handleStageCompletion(ctx, updatingStageIndex, updateRun, updatingStageStatus) } + // Some clusters are still updating. return clusterUpdatingWaitTime, nil } +// handleStageCompletion handles the completion logic when all clusters in a stage are finished. +// Returns the wait time and any error encountered. +func (r *Reconciler) handleStageCompletion( + ctx context.Context, + updatingStageIndex int, + updateRun placementv1beta1.UpdateRunObj, + updatingStageStatus *placementv1beta1.StageUpdatingStatus, +) (time.Duration, error) { + updateRunRef := klog.KObj(updateRun) + + // All the clusters in the stage have been updated. + markUpdateRunWaiting(updateRun, fmt.Sprintf(condition.UpdateRunWaitingMessageFmt, "after-stage", updatingStageStatus.StageName)) + markStageUpdatingWaiting(updatingStageStatus, updateRun.GetGeneration(), "All clusters in the stage are updated, waiting for after-stage tasks to complete") + klog.V(2).InfoS("The stage has finished all cluster updating", "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) + // Check if the after stage tasks are ready. + approved, waitTime, err := r.checkAfterStageTasksStatus(ctx, updatingStageIndex, updateRun) + if err != nil { + return 0, err + } + if approved { + markUpdateRunProgressing(updateRun) + markStageUpdatingSucceeded(updatingStageStatus, updateRun.GetGeneration()) + // No need to wait to get to the next stage. + return 0, nil + } + // The after stage tasks are not ready yet. + if waitTime < 0 { + waitTime = stageUpdatingWaitTime + } + return waitTime, nil +} + // executeDeleteStage executes the delete stage by deleting the bindings. func (r *Reconciler) executeDeleteStage( ctx context.Context, diff --git a/pkg/controllers/updaterun/execution_integration_test.go b/pkg/controllers/updaterun/execution_integration_test.go index 61764f378..3d2af3627 100644 --- a/pkg/controllers/updaterun/execution_integration_test.go +++ b/pkg/controllers/updaterun/execution_integration_test.go @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + promclient "github.com/prometheus/client_model/go" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -189,7 +190,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { It("Should not start rolling out 1st stage", func() { By("Validating the 1st clusterResourceBinding is not updated to Bound") binding := resourceBindings[numTargetClusters-1] // cluster-9 - validateNotBoundBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + validateNotBoundBindingState(ctx, binding) By("Validating the 1st stage does not have startTime set") Expect(updateRun.Status.StagesStatus[0].StartTime).Should(BeNil()) @@ -344,7 +345,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) // 1st stage completed, mark progressing condition reason as succeeded and add succeeded condition. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // 2nd stage waiting for before stage tasks. wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) @@ -395,7 +396,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { It("Should not start rolling out 2nd stage", func() { By("Validating the 1st clusterResourceBinding is not updated to Bound") binding := resourceBindings[0] // cluster-0 - validateNotBoundBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) + validateNotBoundBindingState(ctx, binding) By("Validating the 1st stage does not have startTime set") Expect(updateRun.Status.StagesStatus[1].StartTime).Should(BeNil()) @@ -548,7 +549,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) wantStatus.StagesStatus[1].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) - wantStatus.StagesStatus[1].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[1].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) @@ -594,7 +595,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { return fmt.Errorf("binding %s is not deleted", binding.Name) } if !apierrors.IsNotFound(err) { - return fmt.Errorf("Get binding %s does not return a not-found error: %w", binding.Name, err) + return fmt.Errorf("get binding %s does not return a not-found error: %w", binding.Name, err) } } return nil @@ -605,10 +606,10 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { wantStatus.DeletionStageStatus.Clusters[i].Conditions = append(wantStatus.DeletionStageStatus.Clusters[i].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) } // Mark the stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.DeletionStageStatus.Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -691,9 +692,9 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { By("Validating the updateRun has failed") wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, false) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingFailedReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, false)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunFailedReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -864,13 +865,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Validating the 3rd cluster has succeeded and stage waiting for AfterStageTasks") wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -970,13 +971,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1102,13 +1103,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1198,13 +1199,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Validating the 3rd cluster has succeeded and stage waiting for AfterStageTasks") wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1320,7 +1321,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { It("Should not start rolling out", func() { By("Validating the 1st clusterResourceBinding is not updated to Bound") binding := resourceBindings[0] // cluster-0 - validateNotBoundBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + validateNotBoundBindingState(ctx, binding) By("Validating the 1st stage does not have startTime set") Expect(updateRun.Status.StagesStatus[0].StartTime).Should(BeNil()) @@ -1483,13 +1484,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Validating the 1st stage has completed") wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) // Need to have a longer wait time for the test to pass, because of the long wait time specified in the update strategy. timeout = time.Second * 90 @@ -1516,6 +1517,122 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { }, timeout, interval).Should(BeTrue(), "failed to ensure the approvalRequest is not recreated") }) }) + + Context("Cluster staged update run should update clusters one by one - different states (Initialized -> Executed)", Ordered, func() { + var wantMetrics []*promclient.Metric + BeforeAll(func() { + By("Creating a new clusterStagedUpdateRun") + updateRun.Spec.State = placementv1beta1.StateInitialized + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the initialization succeeded and but not execution started") + wantStatus = generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateInitializationSucceededMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not start execution when the state is Initialize", func() { + By("Validating no execution has started") + Consistently(func() bool { + var currentUpdateRun placementv1beta1.ClusterStagedUpdateRun + if err := k8sClient.Get(ctx, types.NamespacedName{Name: updateRun.Name}, ¤tUpdateRun); err != nil { + return false + } + return meta.FindStatusCondition(currentUpdateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) == nil && + meta.FindStatusCondition(currentUpdateRun.Status.StagesStatus[0].Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)) == nil + }, timeout, interval).Should(BeTrue(), "execution has started unexpectedly") + + By("Validating the 1st clusterResourceBinding is updated to NOT Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBoundBindingState(ctx, binding) + }) + + It("Should start execution after changing the state to Execute", func() { + By("Updating the updateRun state to Execute") + updateRun.Spec.State = placementv1beta1.StateExecuted + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has started") + wantStatus = generateExecutionStartedStatus(updateRun, wantStatus) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 1st clusterResourceBinding is updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 1st clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 1st cluster has succeeded and 2nd cluster has started") + wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the 1st stage has startTime set") + Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 2nd cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 2nd clusterResourceBinding is updated to Bound") + binding := resourceBindings[1] // cluster-1 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 2nd clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 2nd cluster has succeeded and 3rd cluster has started") + wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 3rd cluster in the 1st stage as succeeded after marking the binding available and complete the updateRun", func() { + By("Validating the 3rd clusterResourceBinding is updated to Bound") + binding := resourceBindings[2] // cluster-2 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 3rd clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 3rd cluster has succeeded and stage waiting for AfterStageTasks") + wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + // 1st stage completed. + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) + wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) + // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) + // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. + wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason) + wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the 1st stage has endTime set") + Expect(updateRun.Status.StagesStatus[0].EndTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateSucceededMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + }) }) func validateBindingState(ctx context.Context, binding *placementv1beta1.ClusterResourceBinding, resourceSnapshotName string, updateRun *placementv1beta1.ClusterStagedUpdateRun, stage int) { @@ -1548,7 +1665,7 @@ func validateBindingState(ctx context.Context, binding *placementv1beta1.Cluster }, timeout, interval).Should(Succeed(), "failed to validate the binding state") } -func validateNotBoundBindingState(ctx context.Context, binding *placementv1beta1.ClusterResourceBinding, resourceSnapshotName string, updateRun *placementv1beta1.ClusterStagedUpdateRun, stage int) { +func validateNotBoundBindingState(ctx context.Context, binding *placementv1beta1.ClusterResourceBinding) { Consistently(func() error { if err := k8sClient.Get(ctx, types.NamespacedName{Name: binding.Name}, binding); err != nil { return err diff --git a/pkg/controllers/updaterun/initialization.go b/pkg/controllers/updaterun/initialization.go index baa07be2b..60edac847 100644 --- a/pkg/controllers/updaterun/initialization.go +++ b/pkg/controllers/updaterun/initialization.go @@ -619,7 +619,7 @@ func (r *Reconciler) recordInitializationSucceeded(ctx context.Context, updateRu Status: metav1.ConditionTrue, ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunInitializeSucceededReason, - Message: "ClusterStagedUpdateRun initialized successfully", + Message: "The UpdateRun initialized successfully", }) if updateErr := r.Client.Status().Update(ctx, updateRun); updateErr != nil { klog.ErrorS(updateErr, "Failed to update the UpdateRun status as initialized", "updateRun", klog.KObj(updateRun)) diff --git a/pkg/controllers/updaterun/validation_integration_test.go b/pkg/controllers/updaterun/validation_integration_test.go index b1190601e..b0d55f5f2 100644 --- a/pkg/controllers/updaterun/validation_integration_test.go +++ b/pkg/controllers/updaterun/validation_integration_test.go @@ -34,6 +34,7 @@ import ( clusterv1beta1 "github.com/kubefleet-dev/kubefleet/apis/cluster/v1beta1" placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" "github.com/kubefleet-dev/kubefleet/pkg/utils" + "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" ) var _ = Describe("UpdateRun validation tests", func() { @@ -564,7 +565,7 @@ func generateFailedValidationStatus( updateRun *placementv1beta1.ClusterStagedUpdateRun, started *placementv1beta1.UpdateRunStatus, ) *placementv1beta1.UpdateRunStatus { - started.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, false) + started.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunFailedReason) started.Conditions = append(started.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) return started } diff --git a/test/apis/placement/v1beta1/api_validation_integration_test.go b/test/apis/placement/v1beta1/api_validation_integration_test.go index 03aa7895e..fbf25b969 100644 --- a/test/apis/placement/v1beta1/api_validation_integration_test.go +++ b/test/apis/placement/v1beta1/api_validation_integration_test.go @@ -1213,12 +1213,12 @@ var _ = Describe("Test placement v1beta1 API validation", func() { PlacementName: "test-placement", ResourceSnapshotIndex: "1", StagedUpdateStrategyName: "test-strategy", - State: placementv1beta1.StateNotStarted, + State: placementv1beta1.StateInitialized, }, } Expect(hubClient.Create(ctx, &updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateStarted + updateRun.Spec.State = placementv1beta1.StateExecuted Expect(hubClient.Update(ctx, &updateRun)).Should(Succeed()) Expect(hubClient.Delete(ctx, &updateRun)).Should(Succeed()) }) @@ -1823,7 +1823,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateNotStarted, + State: placementv1beta1.StateInitialized, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) @@ -1843,7 +1843,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { }, } Expect(hubClient.Create(ctx, updateRunWithDefaultState)).Should(Succeed()) - Expect(updateRunWithDefaultState.Spec.State).To(Equal(placementv1beta1.StateNotStarted)) + Expect(updateRunWithDefaultState.Spec.State).To(Equal(placementv1beta1.StateInitialized)) Expect(hubClient.Delete(ctx, updateRunWithDefaultState)).Should(Succeed()) }) @@ -1857,12 +1857,12 @@ var _ = Describe("Test placement v1beta1 API validation", func() { }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - Expect(updateRun.Spec.State).To(Equal(placementv1beta1.StateNotStarted)) + Expect(updateRun.Spec.State).To(Equal(placementv1beta1.StateInitialized)) Expect(hubClient.Delete(ctx, updateRun)).Should(Succeed()) }) It("should allow transition from Initialize to Execute", func() { - updateRun.Spec.State = placementv1beta1.StateStarted + updateRun.Spec.State = placementv1beta1.StateExecuted Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) }) @@ -1882,7 +1882,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateStarted, + State: placementv1beta1.StateExecuted, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) @@ -1913,13 +1913,10 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateStarted, + State: placementv1beta1.StateStopped, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - // Transition to Pause state first - updateRun.Spec.State = placementv1beta1.StateStopped - Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) }) AfterEach(func() { @@ -1927,7 +1924,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { }) It("should allow transition from Pause to Execute", func() { - updateRun.Spec.State = placementv1beta1.StateStarted + updateRun.Spec.State = placementv1beta1.StateExecuted Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) }) @@ -1953,7 +1950,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateNotStarted, + State: placementv1beta1.StateInitialized, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) @@ -1971,12 +1968,12 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateStarted, + State: placementv1beta1.StateExecuted, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateNotStarted + updateRun.Spec.State = placementv1beta1.StateInitialized err := hubClient.Update(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) @@ -1989,17 +1986,13 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateStarted, + State: placementv1beta1.StateStopped, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - // Transition to Pause first - updateRun.Spec.State = placementv1beta1.StateStopped - Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) - // Try to transition back to Initialize - updateRun.Spec.State = placementv1beta1.StateNotStarted + updateRun.Spec.State = placementv1beta1.StateInitialized err := hubClient.Update(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) @@ -2017,7 +2010,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateNotStarted + updateRun.Spec.State = placementv1beta1.StateInitialized err := hubClient.Update(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) @@ -2035,7 +2028,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateStarted + updateRun.Spec.State = placementv1beta1.StateExecuted err := hubClient.Update(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index fc171ea48..b521e65be 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -2068,12 +2068,16 @@ func updateRunStageTaskSucceedConditions(generation int64, taskType placementv1b } func updateRunSucceedConditions(generation int64) []metav1.Condition { + initializeCondGeneration := generation + if generation > 1 { + initializeCondGeneration = 1 + } return []metav1.Condition{ { Type: string(placementv1beta1.StagedUpdateRunConditionInitialized), Status: metav1.ConditionTrue, Reason: condition.UpdateRunInitializeSucceededReason, - ObservedGeneration: generation, + ObservedGeneration: initializeCondGeneration, }, { Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), @@ -2090,6 +2094,17 @@ func updateRunSucceedConditions(generation int64) []metav1.Condition { } } +func updateRunInitializedConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + { + Type: string(placementv1beta1.StagedUpdateRunConditionInitialized), + Status: metav1.ConditionTrue, + Reason: condition.UpdateRunInitializeSucceededReason, + ObservedGeneration: generation, + }, + } +} + func clusterStagedUpdateRunStatusSucceededActual( updateRunName string, wantResourceIndex string, @@ -2101,6 +2116,7 @@ func clusterStagedUpdateRunStatusSucceededActual( wantUnscheduledClusters []string, wantCROs map[string][]string, wantROs map[string][]placementv1beta1.NamespacedName, + execute bool, ) func() error { return func() error { updateRun := &placementv1beta1.ClusterStagedUpdateRun{} @@ -2116,9 +2132,15 @@ func clusterStagedUpdateRunStatusSucceededActual( UpdateStrategySnapshot: wantStrategySpec, } - wantStatus.StagesStatus = buildStageUpdatingStatuses(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) - wantStatus.DeletionStageStatus = buildDeletionStageStatus(wantUnscheduledClusters, updateRun) - wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation) + if execute { + wantStatus.StagesStatus = buildStageUpdatingStatuses(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) + wantStatus.DeletionStageStatus = buildDeletionStageStatus(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation) + } else { + wantStatus.StagesStatus = buildStageUpdatingStatusesForInitialized(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) + wantStatus.DeletionStageStatus = buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunInitializedConditions(updateRun.Generation) + } if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) } @@ -2136,6 +2158,7 @@ func stagedUpdateRunStatusSucceededActual( wantUnscheduledClusters []string, wantCROs map[string][]string, wantROs map[string][]placementv1beta1.NamespacedName, + execute bool, ) func() error { return func() error { updateRun := &placementv1beta1.StagedUpdateRun{} @@ -2151,9 +2174,15 @@ func stagedUpdateRunStatusSucceededActual( UpdateStrategySnapshot: wantStrategySpec, } - wantStatus.StagesStatus = buildStageUpdatingStatuses(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) - wantStatus.DeletionStageStatus = buildDeletionStageStatus(wantUnscheduledClusters, updateRun) - wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation) + if execute { + wantStatus.StagesStatus = buildStageUpdatingStatuses(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) + wantStatus.DeletionStageStatus = buildDeletionStageStatus(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation) + } else { + wantStatus.StagesStatus = buildStageUpdatingStatusesForInitialized(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) + wantStatus.DeletionStageStatus = buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunInitializedConditions(updateRun.Generation) + } if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) } @@ -2161,6 +2190,40 @@ func stagedUpdateRunStatusSucceededActual( } } +func buildStageUpdatingStatusesForInitialized( + wantStrategySpec *placementv1beta1.UpdateStrategySpec, + wantSelectedClusters [][]string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, + updateRun placementv1beta1.UpdateRunObj, +) []placementv1beta1.StageUpdatingStatus { + stagesStatus := make([]placementv1beta1.StageUpdatingStatus, len(wantStrategySpec.Stages)) + for i, stage := range wantStrategySpec.Stages { + stagesStatus[i].StageName = stage.Name + stagesStatus[i].Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantSelectedClusters[i])) + for j := range stagesStatus[i].Clusters { + stagesStatus[i].Clusters[j].ClusterName = wantSelectedClusters[i][j] + stagesStatus[i].Clusters[j].ClusterResourceOverrideSnapshots = wantCROs[wantSelectedClusters[i][j]] + stagesStatus[i].Clusters[j].ResourceOverrideSnapshots = wantROs[wantSelectedClusters[i][j]] + } + stagesStatus[i].BeforeStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.BeforeStageTasks)) + for j, task := range stage.BeforeStageTasks { + stagesStatus[i].BeforeStageTaskStatus[j].Type = task.Type + if task.Type == placementv1beta1.StageTaskTypeApproval { + stagesStatus[i].BeforeStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + } + stagesStatus[i].AfterStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.AfterStageTasks)) + for j, task := range stage.AfterStageTasks { + stagesStatus[i].AfterStageTaskStatus[j].Type = task.Type + if task.Type == placementv1beta1.StageTaskTypeApproval { + stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + } + } + return stagesStatus +} + func buildStageUpdatingStatuses( wantStrategySpec *placementv1beta1.UpdateStrategySpec, wantSelectedClusters [][]string, @@ -2202,6 +2265,15 @@ func buildStageUpdatingStatuses( func buildDeletionStageStatus( wantUnscheduledClusters []string, updateRun placementv1beta1.UpdateRunObj, +) *placementv1beta1.StageUpdatingStatus { + deleteStageStatus := buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + deleteStageStatus.Conditions = updateRunStageRolloutSucceedConditions(updateRun.GetGeneration()) + return deleteStageStatus +} + +func buildDeletionStatusWithoutConditions( + wantUnscheduledClusters []string, + updateRun placementv1beta1.UpdateRunObj, ) *placementv1beta1.StageUpdatingStatus { deleteStageStatus := &placementv1beta1.StageUpdatingStatus{ StageName: "kubernetes-fleet.io/deleteStage", @@ -2211,7 +2283,6 @@ func buildDeletionStageStatus( deleteStageStatus.Clusters[i].ClusterName = wantUnscheduledClusters[i] deleteStageStatus.Clusters[i].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.GetGeneration()) } - deleteStageStatus.Conditions = updateRunStageRolloutSucceedConditions(updateRun.GetGeneration()) return deleteStageStatus } diff --git a/test/e2e/cluster_staged_updaterun_test.go b/test/e2e/cluster_staged_updaterun_test.go index 5bae5faa9..c56ccfc6c 100644 --- a/test/e2e/cluster_staged_updaterun_test.go +++ b/test/e2e/cluster_staged_updaterun_test.go @@ -27,6 +27,7 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -154,7 +155,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -230,7 +231,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -313,7 +314,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -334,7 +335,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -377,7 +378,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a new cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateExecuted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -409,7 +410,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -425,7 +426,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a new staged update run with old resourceSnapshotIndex successfully to rollback", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should rollback resources to member-cluster-2 only and completes stage canary", func() { @@ -457,7 +458,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should rollback resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) for idx := range allMemberClusters { configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &oldConfigMap) @@ -538,7 +539,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -559,7 +560,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should rollout resources to member-cluster-1 after approval but not member-cluster-3 and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -592,7 +593,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should still have resources on member-cluster-1 and member-cluster-2 only and completes stage canary", func() { @@ -615,7 +616,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -647,7 +648,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -664,7 +665,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) // need to go through two stages - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil, true) Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -742,7 +743,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should not rollout any resources to member clusters and complete stage canary", func() { @@ -762,7 +763,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[2]}) checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) @@ -795,7 +796,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should still have resources on member-cluster-2 and member-cluster-3 only and completes stage canary", func() { @@ -817,7 +818,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should rollout resources to member-cluster-1 after approval and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -849,7 +850,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -869,7 +870,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should remove resources on member-cluster-1 after approval and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil, true) Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[1], allMemberClusters[2]}) @@ -1025,7 +1026,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -1047,7 +1048,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1134,7 +1135,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should report diff for member-cluster-2 only and completes stage canary", func() { @@ -1149,7 +1150,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should report diff for member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) }) @@ -1250,7 +1251,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Create a staged update run with new resourceSnapshotIndex and verify rollout happens", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex2nd, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateExecuted) // Verify rollout to canary cluster first By("Verify that the new configmap is updated on member-cluster-2 during canary stage") @@ -1264,7 +1265,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) // Verify complete rollout - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) // Verify new configmap is on all member clusters @@ -1324,7 +1325,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should create a staged update run and verify cluster approval request is created", func() { validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) // Verify that cluster approval request is created for canary stage. Eventually(func() error { @@ -1352,7 +1353,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should complete the staged update run after approval", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1421,7 +1422,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Create updateRun and verify resources are rolled out", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) // Approval for AfterStageTasks of canary stage validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) @@ -1429,7 +1430,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { // Approval for BeforeStageTasks of prod stage validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -1557,13 +1558,13 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should complete the cluster staged update run with all 3 clusters updated in parallel", func() { // With maxConcurrency=3, all 3 clusters should be updated in parallel. // Each round waits 15 seconds, so total time should be under 20s. - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunParallelEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1647,14 +1648,14 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should complete the cluster staged update run with all 3 clusters", func() { // Since maxConcurrency=70% each round we process 2 clusters in parallel, // so all 3 clusters should be updated in 2 rounds. // Each round waits 15 seconds, so total time should be under 40s. - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunParallelEventuallyDuration*2, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1665,6 +1666,111 @@ var _ = Describe("test CRP rollout with staged update run", func() { Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) }) }) + + Context("Test resource rollout with staged update run by update run states - (Initialized -> Executed)", Ordered, func() { + updateRunNames := []string{} + var strategy *placementv1beta1.ClusterStagedUpdateStrategy + + BeforeAll(func() { + // Create a test namespace and a configMap inside it on the hub cluster. + createWorkResources() + + // Create the CRP with external rollout strategy. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: workResourceSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") + + // Create the clusterStagedUpdateStrategy. + strategy = createClusterStagedUpdateStrategySucceed(strategyName) + + for i := 0; i < 1; i++ { + updateRunNames = append(updateRunNames, fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + } + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the CRP. + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + + // Remove all the clusterStagedUpdateRuns. + for _, name := range updateRunNames { + ensureClusterStagedUpdateRunDeletion(name) + } + + // Delete the clusterStagedUpdateStrategy. + ensureClusterUpdateRunStrategyDeletion(strategyName) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) + + It("Should have the latest resource snapshot", func() { + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the crp", func() { + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) + }) + + It("Should update crp status as pending rollout", func() { + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", "", ""}, []bool{false, false, false}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + }) + + It("Should create a cluster staged update run successfully", func() { + By("Creating Cluster Staged Update Run in state Initialize") + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateInitialized) + }) + + It("Should not start rollout as the update run is in Initialize state", func() { + By("Member clusters should not have work resources placed") + checkIfRemovedWorkResourcesFromAllMemberClustersConsistently() + + By("Validating the csur status remains in Initialize state") + csurNotStartedActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, false) + Consistently(csurNotStartedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to Initialize updateRun %s", updateRunNames[0]) + }) + + It("Should rollout resources to member-cluster-2 only after update run is in Execute state", func() { + // Update the update run state to Execute + By("Updating the update run state to Execute") + updateClusterStagedUpdateRunState(updateRunNames[0], placementv1beta1.StateExecuted) + + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating crp status as member-cluster-2 updated") + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) + }) + + It("Should update crp status as completed", func() { + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + }) + }) }) // Note that this container cannot run in parallel with other containers. @@ -1732,10 +1838,10 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) By("Creating the first staged update run") - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) By("Validating staged update run has succeeded") - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) By("Validating CRP status as completed") @@ -1783,11 +1889,11 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should create another staged update run for the same CRP", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 2) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should complete the second staged update run and complete the CRP", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1], allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1], allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames[1:], @@ -1831,11 +1937,11 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should complete the staged update run, complete CRP, and rollout resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, @@ -1874,11 +1980,11 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateExecuted) }) It("Should complete the staged update run, complete CRP, and rollout updated resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex2nd, true, allMemberClusterNames, @@ -1913,11 +2019,11 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should complete the staged update run, complete CRP, and re-place resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, @@ -2059,12 +2165,13 @@ func validateLatestClusterResourceSnapshot(crpName, wantResourceSnapshotIndex st }, eventuallyDuration, eventuallyInterval).Should(Equal(wantResourceSnapshotIndex), "Resource snapshot index does not match") } -func createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex, strategyName string) { +func createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex, strategyName string, state placementv1beta1.State) { updateRun := &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ + State: state, PlacementName: crpName, ResourceSnapshotIndex: resourceSnapshotIndex, StagedUpdateStrategyName: strategyName, @@ -2079,6 +2186,7 @@ func createClusterStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNam Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ + State: placementv1beta1.StateExecuted, PlacementName: crpName, StagedUpdateStrategyName: strategyName, }, @@ -2086,6 +2194,21 @@ func createClusterStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNam Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create ClusterStagedUpdateRun %s", updateRunName) } +func updateClusterStagedUpdateRunState(updateRunName string, state placementv1beta1.State) { + Eventually(func() error { + updateRun := &placementv1beta1.ClusterStagedUpdateRun{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName}, updateRun); err != nil { + return fmt.Errorf("failed to get ClusterStagedUpdateRun %s", updateRunName) + } + + updateRun.Spec.State = state + if err := hubClient.Update(ctx, updateRun); err != nil { + return fmt.Errorf("failed to update ClusterStagedUpdateRun %s", updateRunName) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update ClusterStagedUpdateRun %s state to %s", updateRunName, state) +} + func validateAndApproveClusterApprovalRequests(updateRunName, stageName, approvalRequestNameFmt string) { Eventually(func() error { appReqList := &placementv1beta1.ClusterApprovalRequestList{} diff --git a/test/e2e/staged_updaterun_test.go b/test/e2e/staged_updaterun_test.go index 60dcfe885..95e9dffe2 100644 --- a/test/e2e/staged_updaterun_test.go +++ b/test/e2e/staged_updaterun_test.go @@ -145,7 +145,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -219,7 +219,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -300,7 +300,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -321,7 +321,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -364,7 +364,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a new staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex2nd, strategyName) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateExecuted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -396,7 +396,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -412,7 +412,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a new staged update run with old resourceSnapshotIndex successfully to rollback", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should rollback resources to member-cluster-2 only and completes stage canary", func() { @@ -444,7 +444,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should rollback resources to member-cluster-1 and member-cluster-3 after approval and complete the staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) for idx := range allMemberClusters { configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &oldConfigMap) @@ -523,7 +523,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -544,7 +544,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should rollout resources to member-cluster-1 after approval but not member-cluster-3 and complete the staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -577,7 +577,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should still have resources on member-cluster-1 and member-cluster-2 only and completes stage canary", func() { @@ -600,7 +600,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should rollout resources to member-cluster-3 after approval and complete the staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -632,7 +632,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -649,7 +649,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) // need to go through two stages - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil, true) Eventually(surSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[2]) checkIfRemovedConfigMapFromMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -725,7 +725,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should not rollout any resources to member clusters and complete stage canary", func() { @@ -745,7 +745,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[2]}) checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) @@ -778,7 +778,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should still have resources on member-cluster-2 and member-cluster-3 only and completes stage canary", func() { @@ -800,7 +800,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should rollout resources to member-cluster-1 after approval and complete the staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -832,7 +832,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -852,7 +852,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should remove resources on member-cluster-1 after approval and complete the cluster staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil, true) Eventually(surSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[2]) checkIfRemovedConfigMapFromMemberClusters([]*framework.Cluster{allMemberClusters[0]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[1], allMemberClusters[2]}) @@ -980,7 +980,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -1003,7 +1003,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, wantROs) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, wantROs, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1084,7 +1084,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should report diff for member-cluster-2 only and completes stage canary", func() { @@ -1099,7 +1099,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should report diff for member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) }) @@ -1198,7 +1198,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Create a staged update run with new resourceSnapshotIndex and verify rollout happens", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex2nd, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateExecuted) // Verify rollout to canary cluster first. By("Verify that the new configmap is updated on member-cluster-2 during canary stage") @@ -1212,7 +1212,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) // Verify complete rollout. - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) // Verify new configmap is on all member clusters. @@ -1284,7 +1284,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Create updateRun and verify resources are rolled out", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) // Approval for AfterStageTask of canary stage validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) @@ -1292,7 +1292,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem // Approval for BeforeStageTask of prod stage validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -1419,13 +1419,13 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should complete the staged update run with all 3 clusters updated in parallel", func() { // With maxConcurrency=3, all 3 clusters should be updated in parallel. // Each round waits 15 seconds, so total time should be under 20s. - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunParallelEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1508,14 +1508,14 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) }) It("Should complete the staged update run with all 3 clusters", func() { // Since maxConcurrency=70% each round we process 2 clusters in parallel, // so all 3 clusters should be updated in 2 rounds. // Each round waits 15 seconds, so total time should be under 40s. - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunParallelEventuallyDuration*2, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1526,6 +1526,109 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) }) }) + + Context("Test resource rollout with staged update run by update run states - (Initialize -> Execute)", Ordered, func() { + updateRunNames := []string{} + var strategy *placementv1beta1.StagedUpdateStrategy + + BeforeAll(func() { + // Create the RP with external rollout strategy. + rp := &placementv1beta1.ResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: rpName, + Namespace: testNamespace, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: configMapSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, rp)).To(Succeed(), "Failed to create RP") + + // Create the stagedUpdateStrategy. + strategy = createStagedUpdateStrategySucceed(strategyName, testNamespace) + + for i := 0; i < 3; i++ { + updateRunNames = append(updateRunNames, fmt.Sprintf(stagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + } + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the RP. + ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: testNamespace}, allMemberClusters) + + // Remove all the stagedUpdateRuns. + for _, name := range updateRunNames { + ensureStagedUpdateRunDeletion(name, testNamespace) + } + + // Delete the stagedUpdateStrategy. + ensureStagedUpdateRunStrategyDeletion(strategyName, testNamespace) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedConfigMapFromAllMemberClustersConsistently) + + It("Should have the latest resource snapshot", func() { + validateLatestResourceSnapshot(rpName, testNamespace, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the rp", func() { + validateLatestSchedulingPolicySnapshot(rpName, testNamespace, policySnapshotIndex1st, 3) + }) + + It("Should update rp status as pending rollout", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", "", ""}, []bool{false, false, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + }) + + It("Should create a staged update run successfully", func() { + By("Creating staged update run in Initialize state") + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateInitialized) + }) + + It("Should not start rollout as the update run is in Initialize state", func() { + By("Member clusters should not have work resources placed") + checkIfRemovedConfigMapFromAllMemberClustersConsistently() + + By("Validating the sur status remains in Initialize state") + surNotStartedActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, false) + Consistently(surNotStartedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to Initialize updateRun %s/%s ", testNamespace, updateRunNames[0]) + }) + + It("Should rollout resources to member-cluster-2 only after update run is in Execute state", func() { + // Update the update run state to Execute. + By("Updating the update run state to Execute") + updateStagedUpdateRunState(updateRunNames[0], testNamespace, placementv1beta1.StateExecuted) + + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating crp status as member-cluster-2 updated") + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + }) + + It("Should rollout resources to all the members and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) + Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) + }) + + It("Should update rp status as completed", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + }) + }) }) func createStagedUpdateStrategySucceed(strategyName, namespace string) *placementv1beta1.StagedUpdateStrategy { @@ -1621,7 +1724,7 @@ func validateLatestResourceSnapshot(rpName, namespace, wantResourceSnapshotIndex }, eventuallyDuration, eventuallyInterval).Should(Equal(wantResourceSnapshotIndex), "Resource snapshot index does not match") } -func createStagedUpdateRunSucceed(updateRunName, namespace, rpName, resourceSnapshotIndex, strategyName string) { +func createStagedUpdateRunSucceed(updateRunName, namespace, rpName, resourceSnapshotIndex, strategyName string, state placementv1beta1.State) { updateRun := &placementv1beta1.StagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, @@ -1629,6 +1732,7 @@ func createStagedUpdateRunSucceed(updateRunName, namespace, rpName, resourceSnap }, Spec: placementv1beta1.UpdateRunSpec{ PlacementName: rpName, + State: state, ResourceSnapshotIndex: resourceSnapshotIndex, StagedUpdateStrategyName: strategyName, }, @@ -1643,6 +1747,7 @@ func createStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunName, name Namespace: namespace, }, Spec: placementv1beta1.UpdateRunSpec{ + State: placementv1beta1.StateExecuted, PlacementName: rpName, StagedUpdateStrategyName: strategyName, }, @@ -1650,6 +1755,21 @@ func createStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunName, name Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create StagedUpdateRun %s", updateRunName) } +func updateStagedUpdateRunState(updateRunName, namespace string, state placementv1beta1.State) { + Eventually(func() error { + updateRun := &placementv1beta1.StagedUpdateRun{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName, Namespace: namespace}, updateRun); err != nil { + return fmt.Errorf("failed to get StagedUpdateRun %s", updateRunName) + } + + updateRun.Spec.State = state + if err := hubClient.Update(ctx, updateRun); err != nil { + return fmt.Errorf("failed to update StagedUpdateRun %s", updateRunName) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update StagedUpdateRun %s to state %s", updateRunName, state) +} + func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stageName, approvalRequestNameFmt string) { Eventually(func() error { appReqList := &placementv1beta1.ApprovalRequestList{} From 83bc664c3f623b3e10c43ec059931b8f255ff65b Mon Sep 17 00:00:00 2001 From: Zhiying Lin <54013513+zhiying-lin@users.noreply.github.com> Date: Thu, 4 Dec 2025 09:14:47 +0800 Subject: [PATCH 07/16] fix: use the uncached client when validating override (#351) Signed-off-by: Zhiying Lin --- .../clusterresourceoverride_validating_webhook.go | 8 +++++--- .../resourceoverride_validating_webhook.go | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/webhook/clusterresourceoverride/clusterresourceoverride_validating_webhook.go b/pkg/webhook/clusterresourceoverride/clusterresourceoverride_validating_webhook.go index e83fff3de..89fca6b2e 100644 --- a/pkg/webhook/clusterresourceoverride/clusterresourceoverride_validating_webhook.go +++ b/pkg/webhook/clusterresourceoverride/clusterresourceoverride_validating_webhook.go @@ -40,14 +40,16 @@ var ( ) type clusterResourceOverrideValidator struct { - client client.Client + // Note: we have to use the uncached client here to avoid getting stale data + // since we need to guarantee that a resource cannot be selected by multiple overrides. + client client.Reader decoder webhook.AdmissionDecoder } // Add registers the webhook for K8s bulit-in object types. func Add(mgr manager.Manager) error { hookServer := mgr.GetWebhookServer() - hookServer.Register(ValidationPath, &webhook.Admission{Handler: &clusterResourceOverrideValidator{mgr.GetClient(), admission.NewDecoder(mgr.GetScheme())}}) + hookServer.Register(ValidationPath, &webhook.Admission{Handler: &clusterResourceOverrideValidator{mgr.GetAPIReader(), admission.NewDecoder(mgr.GetScheme())}}) return nil } @@ -80,7 +82,7 @@ func (v *clusterResourceOverrideValidator) Handle(ctx context.Context, req admis } // listClusterResourceOverride returns a list of cluster resource overrides. -func listClusterResourceOverride(ctx context.Context, client client.Client) (*placementv1beta1.ClusterResourceOverrideList, error) { +func listClusterResourceOverride(ctx context.Context, client client.Reader) (*placementv1beta1.ClusterResourceOverrideList, error) { croList := &placementv1beta1.ClusterResourceOverrideList{} if err := client.List(ctx, croList); err != nil { klog.ErrorS(err, "Failed to list clusterResourceOverrides when validating") diff --git a/pkg/webhook/resourceoverride/resourceoverride_validating_webhook.go b/pkg/webhook/resourceoverride/resourceoverride_validating_webhook.go index d6251ec5f..10cadc956 100644 --- a/pkg/webhook/resourceoverride/resourceoverride_validating_webhook.go +++ b/pkg/webhook/resourceoverride/resourceoverride_validating_webhook.go @@ -40,14 +40,16 @@ var ( ) type resourceOverrideValidator struct { - client client.Client + // Note: we have to use the uncached client here to avoid getting stale data + // since we need to guarantee that a resource cannot be selected by multiple overrides. + client client.Reader decoder webhook.AdmissionDecoder } // Add registers the webhook for K8s bulit-in object types. func Add(mgr manager.Manager) error { hookServer := mgr.GetWebhookServer() - hookServer.Register(ValidationPath, &webhook.Admission{Handler: &resourceOverrideValidator{mgr.GetClient(), admission.NewDecoder(mgr.GetScheme())}}) + hookServer.Register(ValidationPath, &webhook.Admission{Handler: &resourceOverrideValidator{mgr.GetAPIReader(), admission.NewDecoder(mgr.GetScheme())}}) return nil } From ba884f5b3cc7333be0a60da821aa92a12790c6b9 Mon Sep 17 00:00:00 2001 From: Britania Rodriguez Reyes <145056127+britaniar@users.noreply.github.com> Date: Thu, 4 Dec 2025 14:03:44 -0600 Subject: [PATCH 08/16] fix: fix the update run states (#363) --- apis/placement/v1beta1/stageupdate_types.go | 36 ++--- ...etes-fleet.io_clusterstagedupdateruns.yaml | 28 ++-- ....kubernetes-fleet.io_stagedupdateruns.yaml | 28 ++-- pkg/controllers/updaterun/controller.go | 4 +- .../updaterun/controller_integration_test.go | 2 +- .../updaterun/execution_integration_test.go | 10 +- .../api_validation_integration_test.go | 129 ++++-------------- test/e2e/cluster_staged_updaterun_test.go | 56 ++++---- test/e2e/staged_updaterun_test.go | 44 +++--- 9 files changed, 123 insertions(+), 214 deletions(-) diff --git a/apis/placement/v1beta1/stageupdate_types.go b/apis/placement/v1beta1/stageupdate_types.go index bc7a8d212..ddfae056a 100644 --- a/apis/placement/v1beta1/stageupdate_types.go +++ b/apis/placement/v1beta1/stageupdate_types.go @@ -152,30 +152,25 @@ func (c *ClusterStagedUpdateRun) SetUpdateRunStatus(status UpdateRunStatus) { type State string const ( - // StateInitialized describes user intent to initialize but not execute the update run. + // StateInitialize describes user intent to initialize but not run the update run. // This is the default state when an update run is created. - // Users can subsequently set the state to Execute or Abandon. - StateInitialized State = "Initialize" + // Users can subsequently set the state to Run. + StateInitialize State = "Initialize" - // StateExecuted describes user intent to execute (or resume execution if paused). - // Users can subsequently set the state to Pause or Abandon. - StateExecuted State = "Execute" + // StateRun describes user intent to execute (or resume execution if stopped). + // Users can subsequently set the state to Stop. + StateRun State = "Run" - // StateStopped describes user intent to pause the update run. - // Users can subsequently set the state to Execute or Abandon. - StateStopped State = "Pause" - - // StateAbandoned describes user intent to abandon the update run. - // This is a terminal state; once set, it cannot be changed. - StateAbandoned State = "Abandon" + // StateStop describes user intent to stop the update run. + // Users can subsequently set the state to Run. + StateStop State = "Stop" ) // UpdateRunSpec defines the desired rollout strategy and the snapshot indices of the resources to be updated. // It specifies a stage-by-stage update process across selected clusters for the given ResourcePlacement object. -// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Initialize' && self.state == 'Pause')",message="invalid state transition: cannot transition from Initialize to Pause" -// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Execute' && self.state == 'Initialize')",message="invalid state transition: cannot transition from Execute to Initialize" -// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Pause' && self.state == 'Initialize')",message="invalid state transition: cannot transition from Pause to Initialize" -// +kubebuilder:validation:XValidation:rule="!has(oldSelf.state) || oldSelf.state != 'Abandon' || self.state == 'Abandon'",message="invalid state transition: Abandon is a terminal state and cannot transition to any other state" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Initialize' && self.state == 'Stop')",message="invalid state transition: cannot transition from Initialize to Stop" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Run' && self.state == 'Initialize')",message="invalid state transition: cannot transition from Run to Initialize" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Stop' && self.state == 'Initialize')",message="invalid state transition: cannot transition from Stop to Initialize" type UpdateRunSpec struct { // PlacementName is the name of placement that this update run is applied to. // There can be multiple active update runs for each placement, but @@ -201,12 +196,11 @@ type UpdateRunSpec struct { // State indicates the desired state of the update run. // Initialize: The update run should be initialized but execution should not start (default). - // Execute: The update run should execute or resume execution. - // Pause: The update run should pause execution. - // Abandon: The update run should be abandoned and terminated. + // Run: The update run should execute or resume execution. + // Stop: The update run should stop execution. // +kubebuilder:validation:Optional // +kubebuilder:default=Initialize - // +kubebuilder:validation:Enum=Initialize;Execute;Pause;Abandon + // +kubebuilder:validation:Enum=Initialize;Run;Stop State State `json:"state,omitempty"` } diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml index c95748724..725eb8ddc 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml @@ -1189,14 +1189,12 @@ spec: description: |- State indicates the desired state of the update run. Initialize: The update run should be initialized but execution should not start (default). - Execute: The update run should execute or resume execution. - Pause: The update run should pause execution. - Abandon: The update run should be abandoned and terminated. + Run: The update run should execute or resume execution. + Stop: The update run should stop execution. enum: - Initialize - - Execute - - Pause - - Abandon + - Run + - Stop type: string required: - placementName @@ -1204,21 +1202,15 @@ spec: type: object x-kubernetes-validations: - message: 'invalid state transition: cannot transition from Initialize - to Pause' + to Stop' rule: '!(has(oldSelf.state) && oldSelf.state == ''Initialize'' && self.state - == ''Pause'')' - - message: 'invalid state transition: cannot transition from Execute to - Initialize' - rule: '!(has(oldSelf.state) && oldSelf.state == ''Execute'' && self.state + == ''Stop'')' + - message: 'invalid state transition: cannot transition from Run to Initialize' + rule: '!(has(oldSelf.state) && oldSelf.state == ''Run'' && self.state == ''Initialize'')' - - message: 'invalid state transition: cannot transition from Pause to - Initialize' - rule: '!(has(oldSelf.state) && oldSelf.state == ''Pause'' && self.state + - message: 'invalid state transition: cannot transition from Stop to Initialize' + rule: '!(has(oldSelf.state) && oldSelf.state == ''Stop'' && self.state == ''Initialize'')' - - message: 'invalid state transition: Abandon is a terminal state and - cannot transition to any other state' - rule: '!has(oldSelf.state) || oldSelf.state != ''Abandon'' || self.state - == ''Abandon''' status: description: The observed status of ClusterStagedUpdateRun. properties: diff --git a/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml index abfa39f46..b06ff9829 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml @@ -109,14 +109,12 @@ spec: description: |- State indicates the desired state of the update run. Initialize: The update run should be initialized but execution should not start (default). - Execute: The update run should execute or resume execution. - Pause: The update run should pause execution. - Abandon: The update run should be abandoned and terminated. + Run: The update run should execute or resume execution. + Stop: The update run should stop execution. enum: - Initialize - - Execute - - Pause - - Abandon + - Run + - Stop type: string required: - placementName @@ -124,21 +122,15 @@ spec: type: object x-kubernetes-validations: - message: 'invalid state transition: cannot transition from Initialize - to Pause' + to Stop' rule: '!(has(oldSelf.state) && oldSelf.state == ''Initialize'' && self.state - == ''Pause'')' - - message: 'invalid state transition: cannot transition from Execute to - Initialize' - rule: '!(has(oldSelf.state) && oldSelf.state == ''Execute'' && self.state + == ''Stop'')' + - message: 'invalid state transition: cannot transition from Run to Initialize' + rule: '!(has(oldSelf.state) && oldSelf.state == ''Run'' && self.state == ''Initialize'')' - - message: 'invalid state transition: cannot transition from Pause to - Initialize' - rule: '!(has(oldSelf.state) && oldSelf.state == ''Pause'' && self.state + - message: 'invalid state transition: cannot transition from Stop to Initialize' + rule: '!(has(oldSelf.state) && oldSelf.state == ''Stop'' && self.state == ''Initialize'')' - - message: 'invalid state transition: Abandon is a terminal state and - cannot transition to any other state' - rule: '!has(oldSelf.state) || oldSelf.state != ''Abandon'' || self.state - == ''Abandon''' status: description: The observed status of StagedUpdateRun. properties: diff --git a/pkg/controllers/updaterun/controller.go b/pkg/controllers/updaterun/controller.go index efd1ced2c..eaef8a949 100644 --- a/pkg/controllers/updaterun/controller.go +++ b/pkg/controllers/updaterun/controller.go @@ -130,7 +130,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim } return runtime.Result{}, initErr } - updatingStageIndex = 0 // start from the first stage (typically for Initialize or Execute states). + updatingStageIndex = 0 // start from the first stage (typically for Initialize or Run states). klog.V(2).InfoS("Initialized the updateRun", "state", state, "updateRun", runObjRef) } else { klog.V(2).InfoS("The updateRun is initialized", "state", state, "updateRun", runObjRef) @@ -159,7 +159,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim } // Execute the updateRun. - if state == placementv1beta1.StateExecuted { + if state == placementv1beta1.StateRun { klog.V(2).InfoS("Continue to execute the updateRun", "state", state, "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) finished, waitTime, execErr := r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings) if errors.Is(execErr, errStagedUpdatedAborted) { diff --git a/pkg/controllers/updaterun/controller_integration_test.go b/pkg/controllers/updaterun/controller_integration_test.go index e017d170e..f61af3ab4 100644 --- a/pkg/controllers/updaterun/controller_integration_test.go +++ b/pkg/controllers/updaterun/controller_integration_test.go @@ -351,7 +351,7 @@ func generateTestClusterStagedUpdateRun() *placementv1beta1.ClusterStagedUpdateR PlacementName: testCRPName, ResourceSnapshotIndex: testResourceSnapshotIndex, StagedUpdateStrategyName: testUpdateStrategyName, - State: placementv1beta1.StateExecuted, + State: placementv1beta1.StateRun, }, } } diff --git a/pkg/controllers/updaterun/execution_integration_test.go b/pkg/controllers/updaterun/execution_integration_test.go index 3d2af3627..6c8cd940a 100644 --- a/pkg/controllers/updaterun/execution_integration_test.go +++ b/pkg/controllers/updaterun/execution_integration_test.go @@ -1518,11 +1518,11 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { }) }) - Context("Cluster staged update run should update clusters one by one - different states (Initialized -> Executed)", Ordered, func() { + Context("Cluster staged update run should update clusters one by one - different states (Initialize -> Run)", Ordered, func() { var wantMetrics []*promclient.Metric BeforeAll(func() { By("Creating a new clusterStagedUpdateRun") - updateRun.Spec.State = placementv1beta1.StateInitialized + updateRun.Spec.State = placementv1beta1.StateInitialize Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and but not execution started") @@ -1550,9 +1550,9 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { validateNotBoundBindingState(ctx, binding) }) - It("Should start execution after changing the state to Execute", func() { - By("Updating the updateRun state to Execute") - updateRun.Spec.State = placementv1beta1.StateExecuted + It("Should start execution after changing the state to Run", func() { + By("Updating the updateRun state to Run") + updateRun.Spec.State = placementv1beta1.StateRun Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") By("Validating the execution has started") diff --git a/test/apis/placement/v1beta1/api_validation_integration_test.go b/test/apis/placement/v1beta1/api_validation_integration_test.go index fbf25b969..25a13d3d1 100644 --- a/test/apis/placement/v1beta1/api_validation_integration_test.go +++ b/test/apis/placement/v1beta1/api_validation_integration_test.go @@ -1213,12 +1213,12 @@ var _ = Describe("Test placement v1beta1 API validation", func() { PlacementName: "test-placement", ResourceSnapshotIndex: "1", StagedUpdateStrategyName: "test-strategy", - State: placementv1beta1.StateInitialized, + State: placementv1beta1.StateInitialize, }, } Expect(hubClient.Create(ctx, &updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateExecuted + updateRun.Spec.State = placementv1beta1.StateRun Expect(hubClient.Update(ctx, &updateRun)).Should(Succeed()) Expect(hubClient.Delete(ctx, &updateRun)).Should(Succeed()) }) @@ -1823,7 +1823,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateInitialized, + State: placementv1beta1.StateInitialize, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) @@ -1839,11 +1839,11 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: "unspecfied-state-update-run-" + fmt.Sprintf("%d", GinkgoParallelProcess()), }, Spec: placementv1beta1.UpdateRunSpec{ - // State not specified - should default to Initialize + // State not specified - should default to Initialize. }, } Expect(hubClient.Create(ctx, updateRunWithDefaultState)).Should(Succeed()) - Expect(updateRunWithDefaultState.Spec.State).To(Equal(placementv1beta1.StateInitialized)) + Expect(updateRunWithDefaultState.Spec.State).To(Equal(placementv1beta1.StateInitialize)) Expect(hubClient.Delete(ctx, updateRunWithDefaultState)).Should(Succeed()) }) @@ -1857,22 +1857,17 @@ var _ = Describe("Test placement v1beta1 API validation", func() { }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - Expect(updateRun.Spec.State).To(Equal(placementv1beta1.StateInitialized)) + Expect(updateRun.Spec.State).To(Equal(placementv1beta1.StateInitialize)) Expect(hubClient.Delete(ctx, updateRun)).Should(Succeed()) }) - It("should allow transition from Initialize to Execute", func() { - updateRun.Spec.State = placementv1beta1.StateExecuted - Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) - }) - - It("should allow transition from Initialize to Abandon", func() { - updateRun.Spec.State = placementv1beta1.StateAbandoned + It("should allow transition from Initialize to Run", func() { + updateRun.Spec.State = placementv1beta1.StateRun Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) }) }) - Context("Test ClusterStagedUpdateRun State API validation - valid Execute state transitions", func() { + Context("Test ClusterStagedUpdateRun State API validation - valid Run state transitions", func() { var updateRun *placementv1beta1.ClusterStagedUpdateRun updateRunName := fmt.Sprintf(validupdateRunNameTemplate, GinkgoParallelProcess()) @@ -1882,7 +1877,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateExecuted, + State: placementv1beta1.StateRun, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) @@ -1892,18 +1887,13 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Expect(hubClient.Delete(ctx, updateRun)).Should(Succeed()) }) - It("should allow transition from Execute to Pause", func() { - updateRun.Spec.State = placementv1beta1.StateStopped - Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) - }) - - It("should allow transition from Execute to Abandon", func() { - updateRun.Spec.State = placementv1beta1.StateAbandoned + It("should allow transition from Run to Stop", func() { + updateRun.Spec.State = placementv1beta1.StateStop Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) }) }) - Context("Test ClusterStagedUpdateRun State API validation - valid Pause state transitions", func() { + Context("Test ClusterStagedUpdateRun State API validation - valid Stop state transitions", func() { var updateRun *placementv1beta1.ClusterStagedUpdateRun updateRunName := fmt.Sprintf(validupdateRunNameTemplate, GinkgoParallelProcess()) @@ -1913,7 +1903,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateStopped, + State: placementv1beta1.StateStop, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) @@ -1923,13 +1913,8 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Expect(hubClient.Delete(ctx, updateRun)).Should(Succeed()) }) - It("should allow transition from Pause to Execute", func() { - updateRun.Spec.State = placementv1beta1.StateExecuted - Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) - }) - - It("should allow transition from Pause to Abandon", func() { - updateRun.Spec.State = placementv1beta1.StateAbandoned + It("should allow transition from Stop to Run", func() { + updateRun.Spec.State = placementv1beta1.StateRun Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) }) }) @@ -1944,113 +1929,59 @@ var _ = Describe("Test placement v1beta1 API validation", func() { } }) - It("should deny transition from Initialize to Pause", func() { - updateRun = &placementv1beta1.ClusterStagedUpdateRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: updateRunName, - }, - Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateInitialized, - }, - } - Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - - updateRun.Spec.State = placementv1beta1.StateStopped - err := hubClient.Update(ctx, updateRun) - var statusErr *k8sErrors.StatusError - Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Initialize to Pause")) - }) - - It("should deny transition from Execute to Initialize", func() { - updateRun = &placementv1beta1.ClusterStagedUpdateRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: updateRunName, - }, - Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateExecuted, - }, - } - Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - - updateRun.Spec.State = placementv1beta1.StateInitialized - err := hubClient.Update(ctx, updateRun) - var statusErr *k8sErrors.StatusError - Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Execute to Initialize")) - }) - - It("should deny transition from Pause to Initialize", func() { - updateRun = &placementv1beta1.ClusterStagedUpdateRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: updateRunName, - }, - Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateStopped, - }, - } - Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - - // Try to transition back to Initialize - updateRun.Spec.State = placementv1beta1.StateInitialized - err := hubClient.Update(ctx, updateRun) - var statusErr *k8sErrors.StatusError - Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Pause to Initialize")) - }) - - It("should deny transition from Abandon to Initialize", func() { + It("should deny transition from Initialize to Stop", func() { updateRun = &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateAbandoned, + State: placementv1beta1.StateInitialize, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateInitialized + updateRun.Spec.State = placementv1beta1.StateStop err := hubClient.Update(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: Abandon is a terminal state and cannot transition to any other state")) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Initialize to Stop")) }) - It("should deny transition from Abandon to Execute", func() { + It("should deny transition from Run to Initialize", func() { updateRun = &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateAbandoned, + State: placementv1beta1.StateRun, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateExecuted + updateRun.Spec.State = placementv1beta1.StateInitialize err := hubClient.Update(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: Abandon is a terminal state and cannot transition to any other state")) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Run to Initialize")) }) - It("should deny transition from Abandon to Pause", func() { + It("should deny transition from Stop to Initialize", func() { updateRun = &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateAbandoned, + State: placementv1beta1.StateStop, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateStopped + // Try to transition back to Initialize. + updateRun.Spec.State = placementv1beta1.StateInitialize err := hubClient.Update(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: Abandon is a terminal state and cannot transition to any other state")) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Stop to Initialize")) }) }) @@ -2070,7 +2001,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { err := hubClient.Create(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("supported values: \"Initialize\", \"Execute\", \"Pause\", \"Abandon\"")) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("supported values: \"Initialize\", \"Run\", \"Stop\"")) }) }) diff --git a/test/e2e/cluster_staged_updaterun_test.go b/test/e2e/cluster_staged_updaterun_test.go index c56ccfc6c..94b293cf1 100644 --- a/test/e2e/cluster_staged_updaterun_test.go +++ b/test/e2e/cluster_staged_updaterun_test.go @@ -314,7 +314,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -378,7 +378,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a new cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -426,7 +426,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a new staged update run with old resourceSnapshotIndex successfully to rollback", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollback resources to member-cluster-2 only and completes stage canary", func() { @@ -539,7 +539,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -593,7 +593,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on member-cluster-1 and member-cluster-2 only and completes stage canary", func() { @@ -648,7 +648,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -743,7 +743,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should not rollout any resources to member clusters and complete stage canary", func() { @@ -796,7 +796,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on member-cluster-2 and member-cluster-3 only and completes stage canary", func() { @@ -850,7 +850,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -1026,7 +1026,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -1135,7 +1135,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should report diff for member-cluster-2 only and completes stage canary", func() { @@ -1251,7 +1251,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Create a staged update run with new resourceSnapshotIndex and verify rollout happens", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateRun) // Verify rollout to canary cluster first By("Verify that the new configmap is updated on member-cluster-2 during canary stage") @@ -1325,7 +1325,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should create a staged update run and verify cluster approval request is created", func() { validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) // Verify that cluster approval request is created for canary stage. Eventually(func() error { @@ -1422,7 +1422,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Create updateRun and verify resources are rolled out", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) // Approval for AfterStageTasks of canary stage validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) @@ -1558,7 +1558,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the cluster staged update run with all 3 clusters updated in parallel", func() { @@ -1648,7 +1648,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the cluster staged update run with all 3 clusters", func() { @@ -1667,7 +1667,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) }) - Context("Test resource rollout with staged update run by update run states - (Initialized -> Executed)", Ordered, func() { + Context("Test resource rollout with staged update run by update run states - (Initialize -> Run)", Ordered, func() { updateRunNames := []string{} var strategy *placementv1beta1.ClusterStagedUpdateStrategy @@ -1730,7 +1730,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should create a cluster staged update run successfully", func() { By("Creating Cluster Staged Update Run in state Initialize") - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateInitialized) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateInitialize) }) It("Should not start rollout as the update run is in Initialize state", func() { @@ -1742,10 +1742,10 @@ var _ = Describe("test CRP rollout with staged update run", func() { Consistently(csurNotStartedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to Initialize updateRun %s", updateRunNames[0]) }) - It("Should rollout resources to member-cluster-2 only after update run is in Execute state", func() { - // Update the update run state to Execute - By("Updating the update run state to Execute") - updateClusterStagedUpdateRunState(updateRunNames[0], placementv1beta1.StateExecuted) + It("Should rollout resources to member-cluster-2 only after update run is in Run state", func() { + // Update the update run state to Run + By("Updating the update run state to Run") + updateClusterStagedUpdateRunState(updateRunNames[0], placementv1beta1.StateRun) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) @@ -1838,7 +1838,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) By("Creating the first staged update run") - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) By("Validating staged update run has succeeded") csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) @@ -1889,7 +1889,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should create another staged update run for the same CRP", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 2) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the second staged update run and complete the CRP", func() { @@ -1937,7 +1937,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the staged update run, complete CRP, and rollout resources to all member clusters", func() { @@ -1980,7 +1980,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateRun) }) It("Should complete the staged update run, complete CRP, and rollout updated resources to all member clusters", func() { @@ -2019,7 +2019,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the staged update run, complete CRP, and re-place resources to all member clusters", func() { @@ -2186,7 +2186,7 @@ func createClusterStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNam Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateExecuted, + State: placementv1beta1.StateRun, PlacementName: crpName, StagedUpdateStrategyName: strategyName, }, diff --git a/test/e2e/staged_updaterun_test.go b/test/e2e/staged_updaterun_test.go index 95e9dffe2..4be7c9394 100644 --- a/test/e2e/staged_updaterun_test.go +++ b/test/e2e/staged_updaterun_test.go @@ -300,7 +300,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -364,7 +364,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a new staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -412,7 +412,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a new staged update run with old resourceSnapshotIndex successfully to rollback", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollback resources to member-cluster-2 only and completes stage canary", func() { @@ -523,7 +523,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -577,7 +577,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on member-cluster-1 and member-cluster-2 only and completes stage canary", func() { @@ -632,7 +632,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -725,7 +725,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should not rollout any resources to member clusters and complete stage canary", func() { @@ -778,7 +778,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on member-cluster-2 and member-cluster-3 only and completes stage canary", func() { @@ -832,7 +832,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -980,7 +980,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -1084,7 +1084,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should report diff for member-cluster-2 only and completes stage canary", func() { @@ -1198,7 +1198,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Create a staged update run with new resourceSnapshotIndex and verify rollout happens", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateRun) // Verify rollout to canary cluster first. By("Verify that the new configmap is updated on member-cluster-2 during canary stage") @@ -1284,7 +1284,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Create updateRun and verify resources are rolled out", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) // Approval for AfterStageTask of canary stage validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) @@ -1419,7 +1419,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the staged update run with all 3 clusters updated in parallel", func() { @@ -1508,7 +1508,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateExecuted) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the staged update run with all 3 clusters", func() { @@ -1527,7 +1527,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) }) - Context("Test resource rollout with staged update run by update run states - (Initialize -> Execute)", Ordered, func() { + Context("Test resource rollout with staged update run by update run states - (Initialize -> Run)", Ordered, func() { updateRunNames := []string{} var strategy *placementv1beta1.StagedUpdateStrategy @@ -1588,7 +1588,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should create a staged update run successfully", func() { By("Creating staged update run in Initialize state") - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateInitialized) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateInitialize) }) It("Should not start rollout as the update run is in Initialize state", func() { @@ -1600,10 +1600,10 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem Consistently(surNotStartedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to Initialize updateRun %s/%s ", testNamespace, updateRunNames[0]) }) - It("Should rollout resources to member-cluster-2 only after update run is in Execute state", func() { - // Update the update run state to Execute. - By("Updating the update run state to Execute") - updateStagedUpdateRunState(updateRunNames[0], testNamespace, placementv1beta1.StateExecuted) + It("Should rollout resources to member-cluster-2 only after update run is in Run state", func() { + // Update the update run state to Run. + By("Updating the update run state to Run") + updateStagedUpdateRunState(updateRunNames[0], testNamespace, placementv1beta1.StateRun) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) @@ -1747,7 +1747,7 @@ func createStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunName, name Namespace: namespace, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateExecuted, + State: placementv1beta1.StateRun, PlacementName: rpName, StagedUpdateStrategyName: strategyName, }, From 3820bb0c7db43385f53b77c018bbd021a9eb764e Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Fri, 5 Dec 2025 10:19:15 +0800 Subject: [PATCH 09/16] feat: trim work object status if the object exceeds size limit (#357) --- apis/placement/v1beta1/work_types.go | 4 + pkg/controllers/workapplier/status.go | 124 ++++ pkg/controllers/workapplier/status_test.go | 719 +++++++++++++++++++++ pkg/utils/resource/resource.go | 25 + pkg/utils/resource/resource_test.go | 66 ++ 5 files changed, 938 insertions(+) diff --git a/apis/placement/v1beta1/work_types.go b/apis/placement/v1beta1/work_types.go index f4592f3d7..d2b04e6e3 100644 --- a/apis/placement/v1beta1/work_types.go +++ b/apis/placement/v1beta1/work_types.go @@ -54,6 +54,10 @@ const ( // WorkConditionTypeDiffReported reports whether Fleet has successfully reported the // configuration difference between the states in the hub cluster and a member cluster. WorkConditionTypeDiffReported = "DiffReported" + + // WorkConditionTypeStatusTrimmed reports whether the member agent has to trim + // the status data in the Work object due to size constraints. + WorkConditionTypeStatusTrimmed = "StatusTrimmed" ) // This api is copied from https://github.com/kubernetes-sigs/work-api/blob/master/pkg/apis/v1alpha1/work_types.go. diff --git a/pkg/controllers/workapplier/status.go b/pkg/controllers/workapplier/status.go index 1a17ba150..8a9fe2667 100644 --- a/pkg/controllers/workapplier/status.go +++ b/pkg/controllers/workapplier/status.go @@ -28,9 +28,17 @@ import ( fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" "github.com/kubefleet-dev/kubefleet/pkg/utils/controller" + "github.com/kubefleet-dev/kubefleet/pkg/utils/resource" +) + +const ( + WorkStatusTrimmedDueToOversizedStatusReason = "Oversized" + WorkStatusTrimmedDueToOversizedStatusMsgTmpl = "The status data (drift/diff details and back-reported status) has been trimmed due to size constraints (%d bytes over limit %d)" ) // refreshWorkStatus refreshes the status of a Work object based on the processing results of its manifests. +// +// TO-DO (chenyu1): refactor this method a bit to reduce its complexity and enable parallelization. func (r *Reconciler) refreshWorkStatus( ctx context.Context, work *fleetv1beta1.Work, @@ -184,6 +192,47 @@ func (r *Reconciler) refreshWorkStatus( setWorkDiffReportedCondition(work, manifestCount, diffReportedObjectsCount) work.Status.ManifestConditions = rebuiltManifestConds + // Perform a size check before the status update. If the Work object goes over the size limit, trim + // some data from its status to ensure that update ops can go through. + // + // Note (chenyu1): at this moment, for simplicity reasons, the trimming op follows a very simple logic: + // if the size limit is breached, the work applier will summarize all drift/diff details in the status, + // and drop all back-reported status data. More sophisticated trimming logic does obviously exist; here + // the controller prefers the simple version primarily for two reasons: + // + // a) in most of the time, it is rare to reach the size limit: KubeFleet's snapshotting mechanism + // tries to keep the total manifest size in a Work object below 800KB (exceptions do exist), which leaves ~600KB + // space for the status data. The work applier reports for each manifest two conditions at most in the + // status (which are all quite small in size), plus the drift/diff details and the back-reported status + // (if applicable); considering the observation that drifts/diffs are not common and their details are usually small + // (just a JSON path plus the before/after values), and the observation that most Kubernetes objects + // only have a few KBs of status data and not all API types need status back-reporting, most of the time + // the Work object should have enough space for status data without trimming; + // b) performing more fine-grained, selective trimming can be a very CPU and memory intensive (e.g. + // various serialization calls) and complex process, and it is difficult to yield optimal results + // even with best efforts. + // + // TO-DO (chenyu1): re-visit this part of the code and evaluate the need for more fine-grained sharding + // if we have users that do use placements of a large collection of manifests and/or very large objects + // with drift/diff detection and status back-reporting on. + // + // TO-DO (chenyu1): evaluate if we need to impose more strict size limits on the manifests to ensure that + // Work objects (almost) always have enough space for status data. + sizeDeltaBytes, err := resource.CalculateSizeDeltaOverLimitFor(work, resource.DefaultObjSizeLimitWithPaddingBytes) + if err != nil { + // Normally this should never occur. + klog.ErrorS(err, "Failed to check Work object size before status update", "work", klog.KObj(work)) + wrappedErr := fmt.Errorf("failed to check work object size before status update: %w", err) + return controller.NewUnexpectedBehaviorError(wrappedErr) + } + if sizeDeltaBytes > 0 { + klog.V(2).InfoS("Must trim status data as the work object has grown over its size limit", + "work", klog.KObj(work), + "sizeDeltaBytes", sizeDeltaBytes, "sizeLimitBytes", resource.DefaultObjSizeLimitWithPaddingBytes) + trimWorkStatusDataWhenOversized(work) + } + setWorkStatusTrimmedCondition(work, sizeDeltaBytes, resource.DefaultObjSizeLimitWithPaddingBytes) + // Update the Work object status. if err := r.hubClient.Status().Update(ctx, work); err != nil { return controller.NewAPIServerError(false, err) @@ -643,3 +692,78 @@ func prepareRebuiltManifestCondQIdx(bundles []*manifestProcessingBundle) map[str } return rebuiltManifestCondQIdx } + +// trimWorkStatusDataWhenOversized trims some data from the Work object status when the object +// reaches its size limit. +func trimWorkStatusDataWhenOversized(work *fleetv1beta1.Work) { + // Trim drift/diff details + back-reported status from the Work object status. + // Replace detailed reportings with a summary if applicable. + for idx := range work.Status.ManifestConditions { + manifestCond := &work.Status.ManifestConditions[idx] + + // Note (chenyu1): check for the second term will always pass; it is added as a sanity check. + if manifestCond.DriftDetails != nil && len(manifestCond.DriftDetails.ObservedDrifts) > 0 { + driftCount := len(manifestCond.DriftDetails.ObservedDrifts) + firstDriftPath := manifestCond.DriftDetails.ObservedDrifts[0].Path + // If there are multiple drifts, report only the path of the first drift plus the count of + // other paths. Also, leave out the specific value differences. + pathSummary := firstDriftPath + if len(manifestCond.DriftDetails.ObservedDrifts) > 1 { + pathSummary = fmt.Sprintf("%s and %d more path(s)", firstDriftPath, driftCount-1) + } + manifestCond.DriftDetails.ObservedDrifts = []fleetv1beta1.PatchDetail{ + { + Path: pathSummary, + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + } + } + + // Note (chenyu1): check for the second term will always pass; it is added as a sanity check. + if manifestCond.DiffDetails != nil && len(manifestCond.DiffDetails.ObservedDiffs) > 0 { + diffCount := len(manifestCond.DiffDetails.ObservedDiffs) + firstDiffPath := manifestCond.DiffDetails.ObservedDiffs[0].Path + // If there are multiple drifts, report only the path of the first drift plus the count of + // other paths. Also, leave out the specific value differences. + pathSummary := firstDiffPath + if len(manifestCond.DiffDetails.ObservedDiffs) > 1 { + pathSummary = fmt.Sprintf("%s and %d more path(s)", firstDiffPath, diffCount-1) + } + manifestCond.DiffDetails.ObservedDiffs = []fleetv1beta1.PatchDetail{ + { + Path: pathSummary, + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + } + } + + manifestCond.BackReportedStatus = nil + } +} + +// setWorkStatusTrimmedCondition sets or removes the StatusTrimmed condition on a Work object +// based on whether the status has been trimmed due to it being oversized. +// +// Note (chenyu1): at this moment, due to limitations on the hub agent controller side (some +// controllers assume that placement related conditions are always set in a specific sequence), +// this StatusTrimmed condition might not be exposed on the placement status properly yet. +func setWorkStatusTrimmedCondition(work *fleetv1beta1.Work, sizeDeltaBytes, sizeLimitBytes int) { + if sizeDeltaBytes <= 0 { + // Drop the StatusTrimmed condition if it exists. + if isCondRemoved := meta.RemoveStatusCondition(&work.Status.Conditions, fleetv1beta1.WorkConditionTypeStatusTrimmed); isCondRemoved { + klog.V(2).InfoS("StatusTrimmed condition removed from Work object status", "work", klog.KObj(work)) + } + return + } + + // Set or update the StatusTrimmed condition. + meta.SetStatusCondition(&work.Status.Conditions, metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeStatusTrimmed, + Status: metav1.ConditionTrue, + Reason: WorkStatusTrimmedDueToOversizedStatusReason, + Message: fmt.Sprintf(WorkStatusTrimmedDueToOversizedStatusMsgTmpl, sizeDeltaBytes, sizeLimitBytes), + ObservedGeneration: work.Generation, + }) +} diff --git a/pkg/controllers/workapplier/status_test.go b/pkg/controllers/workapplier/status_test.go index a8d9f3a96..6fca9b7fe 100644 --- a/pkg/controllers/workapplier/status_test.go +++ b/pkg/controllers/workapplier/status_test.go @@ -26,6 +26,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" @@ -2030,3 +2031,721 @@ func TestSetWorkDiffReportedCondition(t *testing.T) { }) } } + +// TestTrimWorkStatusDataWhenOversized tests the trimWorkStatusDataWhenOversized function. +func TestTrimWorkStatusDataWhenOversized(t *testing.T) { + now := metav1.Now() + + testCases := []struct { + name string + work *fleetv1beta1.Work + wantWorkStatus fleetv1beta1.WorkStatus + }{ + { + name: "no drift/diff data, no back-reported status", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAvailableReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAvailableReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + }, + }, + }, + }, + }, + }, + { + name: "trim drifts (single drift)", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservationTime: now, + FirstDriftedObservedTime: now, + ObservedInMemberClusterGeneration: 1, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservationTime: now, + FirstDriftedObservedTime: now, + ObservedInMemberClusterGeneration: 1, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + }, + }, + }, + }, + }, + }, + { + name: "trim drifts (multiple drifts)", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservationTime: now, + FirstDriftedObservedTime: now, + ObservedInMemberClusterGeneration: 1, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s-1", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + { + Path: fmt.Sprintf("/metadata/labels/%s-2", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservationTime: now, + FirstDriftedObservedTime: now, + ObservedInMemberClusterGeneration: 1, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s-1 and %d more path(s)", dummyLabelKey, 1), + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + }, + }, + }, + }, + }, + }, + { + name: "trim diffs (single diff)", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservationTime: now, + FirstDiffedObservedTime: now, + ObservedInMemberClusterGeneration: ptr.To(int64(1)), + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservationTime: now, + FirstDiffedObservedTime: now, + ObservedInMemberClusterGeneration: ptr.To(int64(1)), + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + }, + }, + }, + }, + }, + }, + { + name: "trim diffs (multiple diffs)", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservationTime: now, + FirstDiffedObservedTime: now, + ObservedInMemberClusterGeneration: ptr.To(int64(1)), + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s-1", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + { + Path: fmt.Sprintf("/metadata/labels/%s-2", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservationTime: now, + FirstDiffedObservedTime: now, + ObservedInMemberClusterGeneration: ptr.To(int64(1)), + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s-1 and %d more path(s)", dummyLabelKey, 1), + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + }, + }, + }, + }, + }, + }, + { + name: "trim back-reported status", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAvailableReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + }, + }, + BackReportedStatus: &fleetv1beta1.BackReportedStatus{ + ObservationTime: now, + ObservedStatus: runtime.RawExtension{ + Raw: []byte(dummyLabelValue1), + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAvailableReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + }, + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + trimWorkStatusDataWhenOversized(tc.work) + if diff := cmp.Diff(tc.work.Status, tc.wantWorkStatus); diff != "" { + t.Errorf("trimmed work status mismatches (-got, +want):\n%s", diff) + } + }) + } +} + +// TestSetWorkStatusTrimmedCondition tests the setWorkStatusTrimmedCondition function. +func TestSetWorkStatusTrimmedCondition(t *testing.T) { + testCases := []struct { + name string + work *fleetv1beta1.Work + sizeDeltaBytes int + sizeLimitBytes int + wantWorkStatusConditions []metav1.Condition + }{ + { + name: "no trimming needed (size delta <= 0)", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Generation: 1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + }, + }, + }, + sizeDeltaBytes: 0, + sizeLimitBytes: 1024, + wantWorkStatusConditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + }, + }, + { + name: "remove existing StatusTrimmed condition when size delta <= 0", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Generation: 2, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 2, + }, + { + Type: fleetv1beta1.WorkConditionTypeStatusTrimmed, + Status: metav1.ConditionTrue, + Reason: WorkStatusTrimmedDueToOversizedStatusReason, + Message: fmt.Sprintf(WorkStatusTrimmedDueToOversizedStatusMsgTmpl, 500, 1024), + ObservedGeneration: 1, + }, + }, + }, + }, + sizeDeltaBytes: 0, + sizeLimitBytes: 1024, + wantWorkStatusConditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 2, + }, + }, + }, + { + name: "set StatusTrimmed condition when size delta > 0", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Generation: 1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + }, + }, + }, + sizeDeltaBytes: 500, + sizeLimitBytes: 1024, + wantWorkStatusConditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + { + Type: fleetv1beta1.WorkConditionTypeStatusTrimmed, + Status: metav1.ConditionTrue, + Reason: WorkStatusTrimmedDueToOversizedStatusReason, + Message: fmt.Sprintf(WorkStatusTrimmedDueToOversizedStatusMsgTmpl, 500, 1024), + ObservedGeneration: 1, + }, + }, + }, + { + name: "update existing StatusTrimmed condition with new values", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Generation: 2, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 2, + }, + { + Type: fleetv1beta1.WorkConditionTypeStatusTrimmed, + Status: metav1.ConditionTrue, + Reason: WorkStatusTrimmedDueToOversizedStatusReason, + Message: fmt.Sprintf(WorkStatusTrimmedDueToOversizedStatusMsgTmpl, 200, 1024), + ObservedGeneration: 1, + }, + }, + }, + }, + sizeDeltaBytes: 750, + sizeLimitBytes: 2048, + wantWorkStatusConditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 2, + }, + { + Type: fleetv1beta1.WorkConditionTypeStatusTrimmed, + Status: metav1.ConditionTrue, + Reason: WorkStatusTrimmedDueToOversizedStatusReason, + Message: fmt.Sprintf(WorkStatusTrimmedDueToOversizedStatusMsgTmpl, 750, 2048), + ObservedGeneration: 2, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + setWorkStatusTrimmedCondition(tc.work, tc.sizeDeltaBytes, tc.sizeLimitBytes) + if diff := cmp.Diff( + tc.work.Status.Conditions, tc.wantWorkStatusConditions, + ignoreFieldConditionLTTMsg, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("work status conditions mismatches (-got, +want):\n%s", diff) + } + }) + } +} diff --git a/pkg/utils/resource/resource.go b/pkg/utils/resource/resource.go index 0f514d8ab..405a08f97 100644 --- a/pkg/utils/resource/resource.go +++ b/pkg/utils/resource/resource.go @@ -21,6 +21,14 @@ import ( "crypto/sha256" "encoding/json" "fmt" + + "k8s.io/apimachinery/pkg/runtime" +) + +const ( + // etcd has a 1.5 MiB limit for objects by default, and Kubernetes clients might + // reject request entities too large (~2/~3 MiB, depending on the protocol in use). + DefaultObjSizeLimitWithPaddingBytes = 1415578 // 1.35 MiB, or ~1.42 MB. ) // HashOf returns the hash of the resource. @@ -31,3 +39,20 @@ func HashOf(resource any) (string, error) { } return fmt.Sprintf("%x", sha256.Sum256(jsonBytes)), nil } + +// CalculateSizeDeltaOverLimitFor calculates the size delta in bytes of a given object +// over a specified size limit. It returns a positive value if the object size exceeds +// the limit or a negative value if the object size is below the limit. +// +// This utility is useful in cases where KubeFleet needs to check if it can create/update +// an object with additional information. +func CalculateSizeDeltaOverLimitFor(obj runtime.Object, sizeLimitBytes int) (int, error) { + jsonBytes, err := json.Marshal(obj) + if err != nil { + return 0, fmt.Errorf("cannot determine object size: %w", err) + } + if sizeLimitBytes < 0 { + return 0, fmt.Errorf("size limit must be non-negative") + } + return len(jsonBytes) - sizeLimitBytes, nil +} diff --git a/pkg/utils/resource/resource_test.go b/pkg/utils/resource/resource_test.go index bfc8418a7..75dc6f854 100644 --- a/pkg/utils/resource/resource_test.go +++ b/pkg/utils/resource/resource_test.go @@ -17,9 +17,13 @@ limitations under the License. package resource import ( + "encoding/json" "testing" + "github.com/google/go-cmp/cmp" placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestHashOf(t *testing.T) { @@ -51,3 +55,65 @@ func TestHashOf(t *testing.T) { }) } } + +// TestCalculateSizeDeltaOverLimitFor tests the CalculateSizeDeltaOverLimitFor function. +func TestCalculateSizeDeltaOverLimitFor(t *testing.T) { + cm := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "app", + Namespace: "default", + }, + Data: map[string]string{ + "key": "value", + }, + } + cmBytes, err := json.Marshal(cm) + if err != nil { + t.Fatalf("Failed to marshal configMap") + } + cmSizeBytes := len(cmBytes) + + testCases := []struct { + name string + sizeLimitBytes int + wantErred bool + }{ + { + name: "under size limit (negative delta)", + sizeLimitBytes: 10000, + }, + { + name: "over size limit (positive delta)", + sizeLimitBytes: 1, + }, + { + name: "invalid size limit (negative size limit)", + // Invalid size limit. + sizeLimitBytes: -1, + wantErred: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + sizeDeltaBytes, err := CalculateSizeDeltaOverLimitFor(cm, tc.sizeLimitBytes) + + if tc.wantErred { + if err == nil { + t.Fatalf("CalculateSizeDeltaOverLimitFor() error = nil, want erred") + } + return + } + // Note: this test spec uses runtime calculation rather than static values for expected + // size delta comparison as different platforms have slight differences in the serialization process, + // which may produce different sizing results. + if !cmp.Equal(sizeDeltaBytes, cmSizeBytes-tc.sizeLimitBytes) { + t.Errorf("CalculateSizeDeltaOverLimitFor() = %d, want %d", sizeDeltaBytes, cmSizeBytes-tc.sizeLimitBytes) + } + }) + } +} From efedeca52f54bf38a17b1ee2c3850baf99b52022 Mon Sep 17 00:00:00 2001 From: Britania Rodriguez Reyes <145056127+britaniar@users.noreply.github.com> Date: Sat, 6 Dec 2025 22:54:07 -0600 Subject: [PATCH 10/16] fix: add label to update run approval object to distinguish between before and after stage tasks (#356) --- apis/placement/v1beta1/commons.go | 11 ++- apis/placement/v1beta1/stageupdate_types.go | 2 + ....kubernetes-fleet.io_approvalrequests.yaml | 1 + ...etes-fleet.io_clusterapprovalrequests.yaml | 1 + pkg/controllers/updaterun/execution.go | 23 +++--- .../updaterun/execution_integration_test.go | 8 ++ pkg/controllers/updaterun/execution_test.go | 18 +++-- test/e2e/cluster_staged_updaterun_test.go | 75 ++++++++++--------- test/e2e/staged_updaterun_test.go | 67 +++++++++-------- 9 files changed, 121 insertions(+), 85 deletions(-) diff --git a/apis/placement/v1beta1/commons.go b/apis/placement/v1beta1/commons.go index 9ced5c1b5..3d118dff0 100644 --- a/apis/placement/v1beta1/commons.go +++ b/apis/placement/v1beta1/commons.go @@ -156,7 +156,10 @@ const ( UpdateRunFinalizer = FleetPrefix + "stagedupdaterun-finalizer" // TargetUpdateRunLabel indicates the target update run on a staged run related object. - TargetUpdateRunLabel = FleetPrefix + "targetupdaterun" + TargetUpdateRunLabel = FleetPrefix + "targetUpdateRun" + + // TaskTypeLabel indicates the task type (before-stage or after-stage) on a staged run related object. + TaskTypeLabel = FleetPrefix + "taskType" // UpdateRunDeleteStageName is the name of delete stage in the staged update run. UpdateRunDeleteStageName = FleetPrefix + "deleteStage" @@ -167,6 +170,12 @@ const ( // TargetUpdatingStageNameLabel indicates the updating stage name on a staged run related object. TargetUpdatingStageNameLabel = FleetPrefix + "targetUpdatingStage" + // BeforeStageTaskLabelValue is the before stage task label value. + BeforeStageTaskLabelValue = "beforeStage" + + // AfterStageTaskLabelValue is the after stage task label value. + AfterStageTaskLabelValue = "afterStage" + // BeforeStageApprovalTaskNameFmt is the format of the before stage approval task name. BeforeStageApprovalTaskNameFmt = "%s-before-%s" diff --git a/apis/placement/v1beta1/stageupdate_types.go b/apis/placement/v1beta1/stageupdate_types.go index ddfae056a..518920728 100644 --- a/apis/placement/v1beta1/stageupdate_types.go +++ b/apis/placement/v1beta1/stageupdate_types.go @@ -673,6 +673,7 @@ type ApprovalRequestObjList interface { // - `TargetUpdateRun`: Points to the cluster staged update run that this approval request is for. // - `TargetStage`: The name of the stage that this approval request is for. // - `IsLatestUpdateRunApproval`: Indicates whether this approval request is the latest one related to this update run. +// - `TaskType`: Indicates whether this approval request is for the before or after stage task. type ClusterApprovalRequest struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -919,6 +920,7 @@ func (s *StagedUpdateStrategyList) GetUpdateStrategyObjs() []UpdateStrategyObj { // - `TargetUpdateRun`: Points to the staged update run that this approval request is for. // - `TargetStage`: The name of the stage that this approval request is for. // - `IsLatestUpdateRunApproval`: Indicates whether this approval request is the latest one related to this update run. +// - `TaskType`: Indicates whether this approval request is for the before or after stage task. type ApprovalRequest struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml b/config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml index 3835fd87c..b16f6ea71 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml @@ -41,6 +41,7 @@ spec: - `TargetUpdateRun`: Points to the staged update run that this approval request is for. - `TargetStage`: The name of the stage that this approval request is for. - `IsLatestUpdateRunApproval`: Indicates whether this approval request is the latest one related to this update run. + - `TaskType`: Indicates whether this approval request is for the before or after stage task. properties: apiVersion: description: |- diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterapprovalrequests.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterapprovalrequests.yaml index 56f4b7552..2333e23f1 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterapprovalrequests.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterapprovalrequests.yaml @@ -172,6 +172,7 @@ spec: - `TargetUpdateRun`: Points to the cluster staged update run that this approval request is for. - `TargetStage`: The name of the stage that this approval request is for. - `IsLatestUpdateRunApproval`: Indicates whether this approval request is the latest one related to this update run. + - `TaskType`: Indicates whether this approval request is for the before or after stage task. properties: apiVersion: description: |- diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index 1180ae34a..74a3a4045 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -83,10 +83,6 @@ func (r *Reconciler) execute( // which would update the lastTransitionTime even though the status hasn't effectively changed. markUpdateRunProgressingIfNotWaitingOrStuck(updateRun) if updatingStageIndex < len(updateRunStatus.StagesStatus) { - maxConcurrency, err := calculateMaxConcurrencyValue(updateRunStatus, updatingStageIndex) - if err != nil { - return false, 0, err - } updatingStageStatus = &updateRunStatus.StagesStatus[updatingStageIndex] approved, err := r.checkBeforeStageTasksStatus(ctx, updatingStageIndex, updateRun) if err != nil { @@ -97,6 +93,10 @@ func (r *Reconciler) execute( markUpdateRunWaiting(updateRun, fmt.Sprintf(condition.UpdateRunWaitingMessageFmt, "before-stage", updatingStageStatus.StageName)) return false, stageUpdatingWaitTime, nil } + maxConcurrency, err := calculateMaxConcurrencyValue(updateRunStatus, updatingStageIndex) + if err != nil { + return false, 0, err + } waitTime, err = r.executeUpdatingStage(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, maxConcurrency) // The execution has not finished yet. return false, waitTime, err @@ -111,17 +111,17 @@ func (r *Reconciler) execute( func (r *Reconciler) checkBeforeStageTasksStatus(ctx context.Context, updatingStageIndex int, updateRun placementv1beta1.UpdateRunObj) (bool, error) { updateRunRef := klog.KObj(updateRun) updateRunStatus := updateRun.GetUpdateRunStatus() - updatingStageStatus := &updateRunStatus.StagesStatus[updatingStageIndex] updatingStage := &updateRunStatus.UpdateStrategySnapshot.Stages[updatingStageIndex] if updatingStage.BeforeStageTasks == nil { klog.V(2).InfoS("There is no before stage task for this stage", "stage", updatingStage.Name, "updateRun", updateRunRef) return true, nil } + updatingStageStatus := &updateRunStatus.StagesStatus[updatingStageIndex] for i, task := range updatingStage.BeforeStageTasks { switch task.Type { case placementv1beta1.StageTaskTypeApproval: - approved, err := r.handleStageApprovalTask(ctx, &updatingStageStatus.BeforeStageTaskStatus[i], updatingStage, updateRun) + approved, err := r.handleStageApprovalTask(ctx, &updatingStageStatus.BeforeStageTaskStatus[i], updatingStage, updateRun, placementv1beta1.BeforeStageTaskLabelValue) if err != nil { return false, err } @@ -418,7 +418,7 @@ func (r *Reconciler) checkAfterStageTasksStatus(ctx context.Context, updatingSta klog.V(2).InfoS("The after stage wait task has completed", "stage", updatingStage.Name, "updateRun", updateRunRef) } case placementv1beta1.StageTaskTypeApproval: - approved, err := r.handleStageApprovalTask(ctx, &updatingStageStatus.AfterStageTaskStatus[i], updatingStage, updateRun) + approved, err := r.handleStageApprovalTask(ctx, &updatingStageStatus.AfterStageTaskStatus[i], updatingStage, updateRun, placementv1beta1.AfterStageTaskLabelValue) if err != nil { return false, -1, err } @@ -440,6 +440,7 @@ func (r *Reconciler) handleStageApprovalTask( stageTaskStatus *placementv1beta1.StageTaskStatus, updatingStage *placementv1beta1.StageConfig, updateRun placementv1beta1.UpdateRunObj, + stageTaskType string, ) (bool, error) { updateRunRef := klog.KObj(updateRun) @@ -450,7 +451,7 @@ func (r *Reconciler) handleStageApprovalTask( } // Check if the approval request has been created. - approvalRequest := buildApprovalRequestObject(types.NamespacedName{Name: stageTaskStatus.ApprovalRequestName, Namespace: updateRun.GetNamespace()}, updatingStage.Name, updateRun.GetName()) + approvalRequest := buildApprovalRequestObject(types.NamespacedName{Name: stageTaskStatus.ApprovalRequestName, Namespace: updateRun.GetNamespace()}, updatingStage.Name, updateRun.GetName(), stageTaskType) requestRef := klog.KObj(approvalRequest) if err := r.Client.Create(ctx, approvalRequest); err != nil { if apierrors.IsAlreadyExists(err) { @@ -618,9 +619,9 @@ func checkClusterUpdateResult( return false, nil } -// buildApprovalRequestObject creates an approval request object for after-stage tasks. +// buildApprovalRequestObject creates an approval request object for before-stage or after-stage tasks. // It returns a ClusterApprovalRequest if namespace is empty, otherwise returns an ApprovalRequest. -func buildApprovalRequestObject(namespacedName types.NamespacedName, stageName, updateRunName string) placementv1beta1.ApprovalRequestObj { +func buildApprovalRequestObject(namespacedName types.NamespacedName, stageName, updateRunName, stageTaskType string) placementv1beta1.ApprovalRequestObj { var approvalRequest placementv1beta1.ApprovalRequestObj if namespacedName.Namespace == "" { approvalRequest = &placementv1beta1.ClusterApprovalRequest{ @@ -629,6 +630,7 @@ func buildApprovalRequestObject(namespacedName types.NamespacedName, stageName, Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: stageTaskType, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -645,6 +647,7 @@ func buildApprovalRequestObject(namespacedName types.NamespacedName, stageName, Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: stageTaskType, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, diff --git a/pkg/controllers/updaterun/execution_integration_test.go b/pkg/controllers/updaterun/execution_integration_test.go index 6c8cd940a..72f532c33 100644 --- a/pkg/controllers/updaterun/execution_integration_test.go +++ b/pkg/controllers/updaterun/execution_integration_test.go @@ -173,6 +173,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -324,6 +325,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -379,6 +381,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[1].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -531,6 +534,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[1].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -1085,6 +1089,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -1306,6 +1311,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -1339,6 +1345,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -1441,6 +1448,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, diff --git a/pkg/controllers/updaterun/execution_test.go b/pkg/controllers/updaterun/execution_test.go index 6d8650d60..510720906 100644 --- a/pkg/controllers/updaterun/execution_test.go +++ b/pkg/controllers/updaterun/execution_test.go @@ -346,22 +346,25 @@ func TestBuildApprovalRequestObject(t *testing.T) { namespacedName types.NamespacedName stageName string updateRunName string + stageTaskType string want placementv1beta1.ApprovalRequestObj }{ { name: "should create ClusterApprovalRequest when namespace is empty", namespacedName: types.NamespacedName{ - Name: "test-approval-request", + Name: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, "test-update-run", "test-stage"), Namespace: "", }, stageName: "test-stage", updateRunName: "test-update-run", + stageTaskType: placementv1beta1.BeforeStageTaskLabelValue, want: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-approval-request", + Name: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, "test-update-run", "test-stage"), Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: "test-stage", placementv1beta1.TargetUpdateRunLabel: "test-update-run", + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -374,18 +377,20 @@ func TestBuildApprovalRequestObject(t *testing.T) { { name: "should create namespaced ApprovalRequest when namespace is provided", namespacedName: types.NamespacedName{ - Name: "test-approval-request", + Name: fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, "test-update-run", "test-stage"), Namespace: "test-namespace", }, stageName: "test-stage", updateRunName: "test-update-run", + stageTaskType: placementv1beta1.AfterStageTaskLabelValue, want: &placementv1beta1.ApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-approval-request", + Name: fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, "test-update-run", "test-stage"), Namespace: "test-namespace", Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: "test-stage", placementv1beta1.TargetUpdateRunLabel: "test-update-run", + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -399,7 +404,7 @@ func TestBuildApprovalRequestObject(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - got := buildApprovalRequestObject(test.namespacedName, test.stageName, test.updateRunName) + got := buildApprovalRequestObject(test.namespacedName, test.stageName, test.updateRunName, test.stageTaskType) // Compare the whole objects using cmp.Diff with ignore options if diff := cmp.Diff(test.want, got); diff != "" { @@ -1035,6 +1040,7 @@ func TestCheckBeforeStageTasksStatus_NegativeCases(t *testing.T) { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: testUpdateRunName, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -1091,6 +1097,7 @@ func TestCheckBeforeStageTasksStatus_NegativeCases(t *testing.T) { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: testUpdateRunName, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -1147,6 +1154,7 @@ func TestCheckBeforeStageTasksStatus_NegativeCases(t *testing.T) { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: testUpdateRunName, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, diff --git a/test/e2e/cluster_staged_updaterun_test.go b/test/e2e/cluster_staged_updaterun_test.go index 94b293cf1..6aa899472 100644 --- a/test/e2e/cluster_staged_updaterun_test.go +++ b/test/e2e/cluster_staged_updaterun_test.go @@ -145,7 +145,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -153,7 +153,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) @@ -217,7 +217,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -229,7 +229,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) @@ -325,7 +325,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -333,7 +333,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) @@ -396,7 +396,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -408,7 +408,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) @@ -444,7 +444,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{resourceSnapshotIndex2nd, resourceSnapshotIndex1st, resourceSnapshotIndex2nd}, []bool{true, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollback resources to prod stage until approved", func() { @@ -456,7 +456,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollback resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) @@ -550,7 +550,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[:2], []string{"", resourceSnapshotIndex1st}, []bool{false, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -558,7 +558,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 after approval but not member-cluster-3 and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) @@ -606,7 +606,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -614,7 +614,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) @@ -658,11 +658,11 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, false, []string{allMemberClusterNames[2]}, []string{resourceSnapshotIndex1st}, []bool{false}, nil, nil) Consistently(crpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should remove resources on member-cluster-1 and member-cluster-2 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) // need to go through two stages csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil, true) @@ -753,7 +753,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[2:], []string{""}, []bool{false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -761,7 +761,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) @@ -808,7 +808,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{false, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to member-cluster-1 until approved", func() { @@ -816,7 +816,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) @@ -860,7 +860,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames[1:], []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true}, nil, nil) Consistently(crpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not remove resources from member-cluster-1 until approved", func() { @@ -868,7 +868,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should remove resources on member-cluster-1 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil, true) Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) @@ -1038,7 +1038,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, wantROs) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to member-cluster-1 and member-cluster-3 until approved", func() { @@ -1046,7 +1046,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) @@ -1144,11 +1144,11 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should report diff for member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) @@ -1259,10 +1259,10 @@ var _ = Describe("test CRP rollout with staged update run", func() { Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s", newConfigMap.Name, allMemberClusterNames[1]) // Approval for AfterStageTasks of canary stage - validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) // Approval for BeforeStageTasks of prod stage - validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) // Verify complete rollout csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) @@ -1333,6 +1333,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ placementv1beta1.TargetUpdatingStageNameLabel: envCanary, placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, }); err != nil { return fmt.Errorf("failed to list approval requests: %w", err) } @@ -1345,11 +1346,11 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should approve after-stage cluster approval request using kubectl-fleet approve plugin for canary stage", func() { - approveClusterApprovalRequest(envCanary, updateRunName) + approveClusterApprovalRequest(envCanary, updateRunName, placementv1beta1.AfterStageTaskLabelValue) }) It("Should approve before-stage cluster approval request using kubectl-fleet approve plugin for prod stage", func() { - approveClusterApprovalRequest(envProd, updateRunName) + approveClusterApprovalRequest(envProd, updateRunName, placementv1beta1.BeforeStageTaskLabelValue) }) It("Should complete the staged update run after approval", func() { @@ -1425,10 +1426,10 @@ var _ = Describe("test CRP rollout with staged update run", func() { createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) // Approval for AfterStageTasks of canary stage - validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) // Approval for BeforeStageTasks of prod stage - validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) @@ -1754,11 +1755,11 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { - validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) @@ -2209,12 +2210,13 @@ func updateClusterStagedUpdateRunState(updateRunName string, state placementv1be }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update ClusterStagedUpdateRun %s state to %s", updateRunName, state) } -func validateAndApproveClusterApprovalRequests(updateRunName, stageName, approvalRequestNameFmt string) { +func validateAndApproveClusterApprovalRequests(updateRunName, stageName, approvalRequestNameFmt, stageTaskType string) { Eventually(func() error { appReqList := &placementv1beta1.ClusterApprovalRequestList{} if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: stageTaskType, }); err != nil { return fmt.Errorf("failed to list approval requests: %w", err) } @@ -2245,7 +2247,7 @@ func updateConfigMapSucceed(newConfigMap *corev1.ConfigMap) { Expect(hubClient.Update(ctx, cm)).To(Succeed(), "Failed to update configmap %s in namespace %s", newConfigMap.Name, newConfigMap.Namespace) } -func approveClusterApprovalRequest(stageName, updateRunName string) { +func approveClusterApprovalRequest(stageName, updateRunName, stageTask string) { var approvalRequestName string // Get the cluster approval request name. @@ -2254,6 +2256,7 @@ func approveClusterApprovalRequest(stageName, updateRunName string) { if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: stageTask, }); err != nil { return fmt.Errorf("failed to list approval requests: %w", err) } diff --git a/test/e2e/staged_updaterun_test.go b/test/e2e/staged_updaterun_test.go index 4be7c9394..447932192 100644 --- a/test/e2e/staged_updaterun_test.go +++ b/test/e2e/staged_updaterun_test.go @@ -135,7 +135,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -143,7 +143,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) @@ -206,7 +206,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") @@ -217,7 +217,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) @@ -311,7 +311,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -319,7 +319,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) @@ -382,7 +382,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -394,7 +394,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) @@ -430,7 +430,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{resourceSnapshotIndex2nd, resourceSnapshotIndex1st, resourceSnapshotIndex2nd}, []bool{true, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollback resources to prod stage until approved", func() { @@ -442,7 +442,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollback resources to member-cluster-1 and member-cluster-3 after approval and complete the staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) @@ -534,7 +534,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[:2], []string{"", resourceSnapshotIndex1st}, []bool{false, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -542,7 +542,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 after approval but not member-cluster-3 and complete the staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) @@ -590,7 +590,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -598,7 +598,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-3 after approval and complete the staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) @@ -642,11 +642,11 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, false, []string{allMemberClusterNames[2]}, []string{resourceSnapshotIndex1st}, []bool{false}, nil, nil) Consistently(rpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should remove resources on member-cluster-1 and member-cluster-2 after approval and complete the staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) // need to go through two stages surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil, true) @@ -735,7 +735,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[2:], []string{""}, []bool{false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to prod stage until approved", func() { @@ -743,7 +743,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) @@ -790,7 +790,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{false, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to member-cluster-1 until approved", func() { @@ -798,7 +798,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 after approval and complete the staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) @@ -842,7 +842,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames[1:], []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true}, nil, nil) Consistently(rpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not remove resources from member-cluster-1 until approved", func() { @@ -850,7 +850,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should remove resources on member-cluster-1 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil, true) Eventually(surSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[2]) @@ -993,7 +993,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, wantROs) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should not rollout resources to member-cluster-1 and member-cluster-3 until approved", func() { @@ -1001,7 +1001,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, wantROs, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) @@ -1093,11 +1093,11 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should report diff for member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) @@ -1206,10 +1206,10 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s", newConfigMap.Name, allMemberClusterNames[1]) // Approval for AfterStageTask of canary stage - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) // Approval for BeforeStageTask of prod stage - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) // Verify complete rollout. surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) @@ -1287,10 +1287,10 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) // Approval for AfterStageTask of canary stage - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) // Approval for BeforeStageTask of prod stage - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) @@ -1612,11 +1612,11 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) It("Should rollout resources to all the members and complete the staged update run successfully", func() { - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) @@ -1770,12 +1770,13 @@ func updateStagedUpdateRunState(updateRunName, namespace string, state placement }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update StagedUpdateRun %s to state %s", updateRunName, state) } -func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stageName, approvalRequestNameFmt string) { +func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stageName, approvalRequestNameFmt, stageTaskType string) { Eventually(func() error { appReqList := &placementv1beta1.ApprovalRequestList{} if err := hubClient.List(ctx, appReqList, client.InNamespace(namespace), client.MatchingLabels{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: stageTaskType, }); err != nil { return fmt.Errorf("failed to list approval requests: %w", err) } From 1753dfb86892e379c829f7bf824cf78763ee8b35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Dec 2025 10:26:55 +0800 Subject: [PATCH 11/16] chore: bump step-security/harden-runner from 2.13.2 to 2.13.3 (#372) Bumps [step-security/harden-runner](https://github.com/step-security/harden-runner) from 2.13.2 to 2.13.3. - [Release notes](https://github.com/step-security/harden-runner/releases) - [Commits](https://github.com/step-security/harden-runner/compare/95d9a5deda9de15063e7595e9719c11c38c90ae2...df199fb7be9f65074067a9eb93f12bb4c5547cf2) --- updated-dependencies: - dependency-name: step-security/harden-runner dependency-version: 2.13.3 dependency-type: direct:production update-type: version-update:semver-patch ... --- .github/workflows/codespell.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index c91f5d2d8..574c3e612 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 + uses: step-security/harden-runner@df199fb7be9f65074067a9eb93f12bb4c5547cf2 # v2.13.3 with: egress-policy: audit From 95016cec98b30d789a20740a8d9e8c8f1d72a6f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Dec 2025 11:33:53 +0800 Subject: [PATCH 12/16] chore: bump actions/checkout from 6.0.0 to 6.0.1 (#371) Bumps [actions/checkout](https://github.com/actions/checkout) from 6.0.0 to 6.0.1. - [Release notes](https://github.com/actions/checkout/releases) - [Commits](https://github.com/actions/checkout/compare/v6...v6.0.1) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 6.0.1 dependency-type: direct:production update-type: version-update:semver-patch ... --- .github/workflows/chart.yml | 2 +- .github/workflows/ci.yml | 4 ++-- .github/workflows/code-lint.yml | 4 ++-- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/codespell.yml | 2 +- .github/workflows/markdown-lint.yml | 2 +- .github/workflows/release.yml | 4 ++-- .github/workflows/trivy.yml | 2 +- .github/workflows/upgrade.yml | 6 +++--- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/chart.yml b/.github/workflows/chart.yml index f1d2a197c..a2d709555 100644 --- a/.github/workflows/chart.yml +++ b/.github/workflows/chart.yml @@ -18,7 +18,7 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 with: submodules: true fetch-depth: 0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45193c1f5..8376f748c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 - name: Set up Ginkgo CLI run: | @@ -91,7 +91,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 - name: Install Ginkgo CLI run: | diff --git a/.github/workflows/code-lint.yml b/.github/workflows/code-lint.yml index a49324aed..ac3906219 100644 --- a/.github/workflows/code-lint.yml +++ b/.github/workflows/code-lint.yml @@ -43,7 +43,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Checkout - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 with: submodules: true @@ -64,7 +64,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 - name: golangci-lint run: make lint diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2536446da..91deab101 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 574c3e612..5730dfc94 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -16,7 +16,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@c2d88d3ecc89a9ef08eebf45d9637801dcee7eb5 # v4.1.7 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v4.1.7 - uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # master with: check_filenames: true diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml index 7c8815f5a..bb2858063 100644 --- a/.github/workflows/markdown-lint.yml +++ b/.github/workflows/markdown-lint.yml @@ -10,7 +10,7 @@ jobs: markdown-link-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 - uses: tcort/github-action-markdown-link-check@v1 with: # this will only show errors in the output diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 57227d1a9..d6fff39de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,7 +30,7 @@ jobs: tag: ${{ steps.export.outputs.tag }} steps: - name: Checkout code - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 - id: export run: | @@ -64,7 +64,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Checkout code - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 - name: Login to ghcr.io uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index ab45f089a..995d8dd19 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -44,7 +44,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Checkout code - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 - name: Login to ${{ env.REGISTRY }} uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef diff --git a/.github/workflows/upgrade.yml b/.github/workflows/upgrade.yml index 365dc1c33..d33864ae8 100644 --- a/.github/workflows/upgrade.yml +++ b/.github/workflows/upgrade.yml @@ -44,7 +44,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 with: # Fetch the history of all branches and tags. # This is needed for the test suite to switch between releases. @@ -127,7 +127,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 with: # Fetch the history of all branches and tags. # This is needed for the test suite to switch between releases. @@ -210,7 +210,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 with: # Fetch the history of all branches and tags. # This is needed for the test suite to switch between releases. From 0f449dc759c144e672fbfa574216912cd3e16922 Mon Sep 17 00:00:00 2001 From: Zhiying Lin <54013513+zhiying-lin@users.noreply.github.com> Date: Mon, 8 Dec 2025 16:14:35 +0800 Subject: [PATCH 13/16] test: fix the TestPickBindingsToRoll ut (#365) Signed-off-by: Zhiying Lin Co-authored-by: Ryan Zhang --- pkg/controllers/rollout/controller_test.go | 342 +++++++++++++-------- 1 file changed, 206 insertions(+), 136 deletions(-) diff --git a/pkg/controllers/rollout/controller_test.go b/pkg/controllers/rollout/controller_test.go index 34277f03f..8ac5b7884 100644 --- a/pkg/controllers/rollout/controller_test.go +++ b/pkg/controllers/rollout/controller_test.go @@ -1218,7 +1218,10 @@ func TestIsBindingReady(t *testing.T) { func TestPickBindingsToRoll(t *testing.T) { tests := map[string]struct { - allBindings []*placementv1beta1.ClusterResourceBinding + // We have to generate the bindings before calling PickBindingsToRoll instead of building them + // during the initialization. + // So that the waitTime is under the control. + allBindingsFunc func() []*placementv1beta1.ClusterResourceBinding latestResourceSnapshotName string crp *placementv1beta1.ClusterResourcePlacement matchedCROs []*placementv1beta1.ClusterResourceOverrideSnapshot @@ -1234,8 +1237,10 @@ func TestPickBindingsToRoll(t *testing.T) { }{ // TODO: add more tests "test scheduled binding to bound, outdated resources and nil overrides - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1253,8 +1258,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test scheduled binding to bound, outdated resources and updated apply strategy - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1276,8 +1283,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test scheduled binding to bound, outdated resources and empty overrides - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1297,8 +1306,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test scheduled binding to bound, outdated resources with overrides matching cluster - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1391,8 +1402,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test scheduled binding to bound, outdated resources with overrides not matching any cluster - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1474,8 +1487,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test scheduled binding to bound, overrides matching cluster - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -1568,8 +1583,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test bound binding with latest resources - rollout not needed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -1581,8 +1598,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantNeedRoll: false, }, "test failed to apply bound binding, outdated resources - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1600,12 +1619,14 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: defaultUnavailablePeriod * time.Second, }, "test one failed to apply bound binding and four failed non ready bound bindings, outdated resources with maxUnavailable specified - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1644,12 +1665,14 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: defaultUnavailablePeriod * time.Second, }, "test three failed to apply bound binding, two ready bound binding, outdated resources with maxUnavailable specified - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1688,12 +1711,14 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: defaultUnavailablePeriod * time.Second, }, "test bound ready bindings, maxUnavailable is set to zero - rollout blocked": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1742,7 +1767,9 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test with no bindings": { - allBindings: []*placementv1beta1.ClusterResourceBinding{}, + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{} + }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", createPlacementPolicyForTest(placementv1beta1.PickNPlacementType, 5), @@ -1752,9 +1779,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test two scheduled bindings, outdated resources - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1777,9 +1806,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test canBeReady bound and scheduled binding - rollout allowed with unavailable period wait time": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1813,9 +1844,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 60 * time.Second, }, "test two unscheduled bindings, maxUnavailable specified - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -1831,8 +1864,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test overrides and the cluster is not found": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-1", matchedCROs: []*placementv1beta1.ClusterResourceOverrideSnapshot{ @@ -1870,10 +1905,12 @@ func TestPickBindingsToRoll(t *testing.T) { }, "test bound bindings with different waitTimes and check the wait time should be the min of them all": { // want the min wait time of bound bindings that are not ready - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3, metav1.Time{Time: now.Add(-35 * time.Second)}), // notReady, waitTime = t - 35s - generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), // notReady, no wait time because it does not have available condition yet, - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-2", cluster2), // Ready + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3, metav1.Time{Time: time.Now().Add(-35 * time.Second)}), // notReady, waitTime = t - 35s + generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), // notReady, no wait time because it does not have available condition yet, + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-2", cluster2), // Ready + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1909,9 +1946,11 @@ func TestPickBindingsToRoll(t *testing.T) { }, "test unscheduled bindings with different waitTimes and check the wait time is correct": { // want the min wait time of unscheduled bindings that are not ready - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2, metav1.Time{Time: now.Add(-1 * time.Minute)}), // NotReady, waitTime = t - 60s - generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3, metav1.Time{Time: now.Add(-35 * time.Second)}), // NotReady, waitTime = t - 35s + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2, metav1.Time{Time: time.Now().Add(-1 * time.Minute)}), // NotReady, waitTime = t - 60s + generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3, metav1.Time{Time: time.Now().Add(-35 * time.Second)}), // NotReady, waitTime = t - 35s + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1933,8 +1972,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 25 * time.Second, // minWaitTime = (t - 35 seconds) - (t - 60 seconds) = 25 seconds }, "test only one bound but is deleting binding - rollout blocked": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + } }, crp: clusterResourcePlacementForTest("test", createPlacementPolicyForTest(placementv1beta1.PickAllPlacementType, 0), @@ -1944,11 +1985,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantNeedRoll: false, }, "test policy change with MaxSurge specified, evict resources on unscheduled cluster - rollout allowed for one scheduled binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), - generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), + generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -1984,11 +2027,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: time.Second, }, "test policy change with MaxUnavailable specified, evict resources on unscheduled cluster - rollout allowed for one unscheduled binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2024,9 +2069,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test resource snapshot change with MaxUnavailable specified, evict resources on ready bound binding - rollout allowed for one ready bound binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2056,9 +2103,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test resource snapshot change with MaxUnavailable specified, evict resource on canBeReady binding - rollout blocked": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2088,9 +2137,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: time.Second, }, "test resource snapshot change with MaxUnavailable specified, evict resources on failed to apply bound binding - rollout allowed for one failed to apply bound binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2120,11 +2171,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: time.Second, }, "test upscale, evict resources from ready bound binding - rollout allowed for two new scheduled bindings": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2161,13 +2214,15 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test upscale with policy change MaxSurge specified, evict resources from canBeReady bound binding - rollout allowed for three new scheduled bindings": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), - generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), + generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2213,11 +2268,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: time.Second, }, "test upscale with resource change MaxUnavailable specified, evict resources from ready bound binding - rollout allowed for old bound and new scheduled bindings": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-2", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-2", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-2", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-2", cluster4), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2257,11 +2314,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test downscale, evict resources from ready unscheduled binding - rollout allowed for one unscheduled binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2290,11 +2349,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test downscale, evict resources from ready bound binding - rollout allowed for two unscheduled bindings to be deleted": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2323,13 +2384,15 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test downscale with policy change, evict unscheduled ready binding - rollout allowed for one unscheduled binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2367,13 +2430,15 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test downscale with policy change, evict unscheduled failed to apply binding - rollout allowed for new scheduled bindings": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2411,11 +2476,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: defaultUnavailablePeriod * time.Second, }, "test downscale with resource snapshot change, evict ready bound binding - rollout allowed for one unscheduled binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2455,11 +2522,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test downscale with resource snapshot change, evict ready unscheduled binding - rollout allowed for one unscheduled binding, one bound binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2519,7 +2588,8 @@ func TestPickBindingsToRoll(t *testing.T) { Name: tt.latestResourceSnapshotName, }, } - gotUpdatedBindings, gotStaleUnselectedBindings, gotUpToDateBoundBindings, gotNeedRoll, gotWaitTime, err := r.pickBindingsToRoll(context.Background(), controller.ConvertCRBArrayToBindingObjs(tt.allBindings), resourceSnapshot, tt.crp, tt.matchedCROs, tt.matchedROs) + allBindings := tt.allBindingsFunc() + gotUpdatedBindings, gotStaleUnselectedBindings, gotUpToDateBoundBindings, gotNeedRoll, gotWaitTime, err := r.pickBindingsToRoll(context.Background(), controller.ConvertCRBArrayToBindingObjs(tt.allBindingsFunc()), resourceSnapshot, tt.crp, tt.matchedCROs, tt.matchedROs) if (err != nil) != (tt.wantErr != nil) || err != nil && !errors.Is(err, tt.wantErr) { t.Fatalf("pickBindingsToRoll() error = %v, wantErr %v", err, tt.wantErr) } @@ -2530,30 +2600,30 @@ func TestPickBindingsToRoll(t *testing.T) { wantTobeUpdatedBindings := make([]toBeUpdatedBinding, len(tt.wantTobeUpdatedBindings)) for i, index := range tt.wantTobeUpdatedBindings { // Unscheduled bindings are only removed in a single rollout cycle. - bindingSpec := tt.allBindings[index].GetBindingSpec() + bindingSpec := allBindings[index].GetBindingSpec() if bindingSpec.State != placementv1beta1.BindingStateUnscheduled { - wantTobeUpdatedBindings[i].currentBinding = tt.allBindings[index] - wantTobeUpdatedBindings[i].desiredBinding = tt.allBindings[index].DeepCopy() + wantTobeUpdatedBindings[i].currentBinding = allBindings[index] + wantTobeUpdatedBindings[i].desiredBinding = allBindings[index].DeepCopy() wantTobeUpdatedBindings[i].desiredBinding.SetBindingSpec(tt.wantDesiredBindingsSpec[index]) } else { - wantTobeUpdatedBindings[i].currentBinding = tt.allBindings[index] + wantTobeUpdatedBindings[i].currentBinding = allBindings[index] } } wantStaleUnselectedBindings := make([]toBeUpdatedBinding, len(tt.wantStaleUnselectedBindings)) for i, index := range tt.wantStaleUnselectedBindings { // Unscheduled bindings are only removed in a single rollout cycle. - bindingSpec := tt.allBindings[index].GetBindingSpec() + bindingSpec := allBindings[index].GetBindingSpec() if bindingSpec.State != placementv1beta1.BindingStateUnscheduled { - wantStaleUnselectedBindings[i].currentBinding = tt.allBindings[index] - wantStaleUnselectedBindings[i].desiredBinding = tt.allBindings[index].DeepCopy() + wantStaleUnselectedBindings[i].currentBinding = allBindings[index] + wantStaleUnselectedBindings[i].desiredBinding = allBindings[index].DeepCopy() wantStaleUnselectedBindings[i].desiredBinding.SetBindingSpec(tt.wantDesiredBindingsSpec[index]) } else { - wantStaleUnselectedBindings[i].currentBinding = tt.allBindings[index] + wantStaleUnselectedBindings[i].currentBinding = allBindings[index] } } wantUpToDateBoundBindings := make([]toBeUpdatedBinding, len(tt.wantUpToDateBoundBindings)) for i, index := range tt.wantUpToDateBoundBindings { - wantUpToDateBoundBindings[i].currentBinding = tt.allBindings[index] + wantUpToDateBoundBindings[i].currentBinding = allBindings[index] } if diff := cmp.Diff(wantTobeUpdatedBindings, gotUpdatedBindings, cmpOptions...); diff != "" { From 58b6de2a8d95ede3c3bf09888b0b3258327ea279 Mon Sep 17 00:00:00 2001 From: Ryan Zhang Date: Mon, 8 Dec 2025 11:31:26 -0800 Subject: [PATCH 14/16] fix: fix the validation webhook (#359) --- cmd/hubagent/main.go | 8 +- pkg/webhook/add_handler.go | 2 +- .../membercluster_validating_webhook.go | 22 ++- .../membercluster_validating_webhook_test.go | 141 ++++++++++++++++++ pkg/webhook/webhook.go | 4 +- test/e2e/join_and_leave_test.go | 49 +----- 6 files changed, 168 insertions(+), 58 deletions(-) create mode 100644 pkg/webhook/membercluster/membercluster_validating_webhook_test.go diff --git a/cmd/hubagent/main.go b/cmd/hubagent/main.go index f896ab71a..42409c0ea 100644 --- a/cmd/hubagent/main.go +++ b/cmd/hubagent/main.go @@ -156,7 +156,8 @@ func main() { if opts.EnableWebhook { whiteListedUsers := strings.Split(opts.WhiteListedUsers, ",") - if err := SetupWebhook(mgr, options.WebhookClientConnectionType(opts.WebhookClientConnectionType), opts.WebhookServiceName, whiteListedUsers, opts.EnableGuardRail, opts.EnableV1Beta1APIs, opts.DenyModifyMemberClusterLabels, opts.EnableWorkload); err != nil { + if err := SetupWebhook(mgr, options.WebhookClientConnectionType(opts.WebhookClientConnectionType), opts.WebhookServiceName, whiteListedUsers, + opts.EnableGuardRail, opts.EnableV1Beta1APIs, opts.DenyModifyMemberClusterLabels, opts.EnableWorkload, opts.NetworkingAgentsEnabled); err != nil { klog.ErrorS(err, "unable to set up webhook") exitWithErrorFunc() } @@ -188,7 +189,8 @@ func main() { } // SetupWebhook generates the webhook cert and then set up the webhook configurator. -func SetupWebhook(mgr manager.Manager, webhookClientConnectionType options.WebhookClientConnectionType, webhookServiceName string, whiteListedUsers []string, enableGuardRail, isFleetV1Beta1API bool, denyModifyMemberClusterLabels bool, enableWorkload bool) error { +func SetupWebhook(mgr manager.Manager, webhookClientConnectionType options.WebhookClientConnectionType, webhookServiceName string, + whiteListedUsers []string, enableGuardRail, isFleetV1Beta1API bool, denyModifyMemberClusterLabels bool, enableWorkload bool, networkingAgentsEnabled bool) error { // Generate self-signed key and crt files in FleetWebhookCertDir for the webhook server to start. w, err := webhook.NewWebhookConfig(mgr, webhookServiceName, FleetWebhookPort, &webhookClientConnectionType, FleetWebhookCertDir, enableGuardRail, denyModifyMemberClusterLabels, enableWorkload) if err != nil { @@ -199,7 +201,7 @@ func SetupWebhook(mgr manager.Manager, webhookClientConnectionType options.Webho klog.ErrorS(err, "unable to add WebhookConfig") return err } - if err = webhook.AddToManager(mgr, whiteListedUsers, denyModifyMemberClusterLabels); err != nil { + if err = webhook.AddToManager(mgr, whiteListedUsers, denyModifyMemberClusterLabels, networkingAgentsEnabled); err != nil { klog.ErrorS(err, "unable to register webhooks to the manager") return err } diff --git a/pkg/webhook/add_handler.go b/pkg/webhook/add_handler.go index 08380e6b8..24f3a6eb8 100644 --- a/pkg/webhook/add_handler.go +++ b/pkg/webhook/add_handler.go @@ -16,13 +16,13 @@ import ( func init() { // AddToManagerFleetResourceValidator is a function to register fleet guard rail resource validator to the webhook server AddToManagerFleetResourceValidator = fleetresourcehandler.Add + AddToManagerMemberclusterValidator = membercluster.Add // AddToManagerFuncs is a list of functions to register webhook validators and mutators to the webhook server AddToManagerFuncs = append(AddToManagerFuncs, clusterresourceplacement.AddMutating) AddToManagerFuncs = append(AddToManagerFuncs, clusterresourceplacement.Add) AddToManagerFuncs = append(AddToManagerFuncs, resourceplacement.Add) AddToManagerFuncs = append(AddToManagerFuncs, pod.Add) AddToManagerFuncs = append(AddToManagerFuncs, replicaset.Add) - AddToManagerFuncs = append(AddToManagerFuncs, membercluster.Add) AddToManagerFuncs = append(AddToManagerFuncs, clusterresourceoverride.Add) AddToManagerFuncs = append(AddToManagerFuncs, resourceoverride.Add) AddToManagerFuncs = append(AddToManagerFuncs, clusterresourceplacementeviction.Add) diff --git a/pkg/webhook/membercluster/membercluster_validating_webhook.go b/pkg/webhook/membercluster/membercluster_validating_webhook.go index 24df38cdd..895797bd2 100644 --- a/pkg/webhook/membercluster/membercluster_validating_webhook.go +++ b/pkg/webhook/membercluster/membercluster_validating_webhook.go @@ -26,15 +26,19 @@ var ( ) type memberClusterValidator struct { - client client.Client - decoder webhook.AdmissionDecoder + client client.Client + decoder webhook.AdmissionDecoder + networkingAgentsEnabled bool } // Add registers the webhook for K8s bulit-in object types. -func Add(mgr manager.Manager) error { +func Add(mgr manager.Manager, networkingAgentsEnabled bool) { hookServer := mgr.GetWebhookServer() - hookServer.Register(ValidationPath, &webhook.Admission{Handler: &memberClusterValidator{client: mgr.GetClient(), decoder: admission.NewDecoder(mgr.GetScheme())}}) - return nil + hookServer.Register(ValidationPath, &webhook.Admission{Handler: &memberClusterValidator{ + client: mgr.GetClient(), + decoder: admission.NewDecoder(mgr.GetScheme()), + networkingAgentsEnabled: networkingAgentsEnabled, + }}) } // Handle memberClusterValidator checks to see if member cluster has valid fields. @@ -50,8 +54,12 @@ func (v *memberClusterValidator) Handle(ctx context.Context, req admission.Reque } if mc.Spec.DeleteOptions != nil && mc.Spec.DeleteOptions.ValidationMode == clusterv1beta1.DeleteValidationModeSkip { - klog.V(2).InfoS("Skipping validation for member cluster DELETE", "memberCluster", mcObjectName) - return admission.Allowed("Skipping validation for member cluster DELETE") + klog.V(2).InfoS("Skipping validation for member cluster DELETE when the validation mode is set to skip", "memberCluster", mcObjectName) + return admission.Allowed("Skipping validation for member cluster DELETE when the validation mode is set to skip") + } + if !v.networkingAgentsEnabled { + klog.V(2).InfoS("Networking agents disabled; skipping ServiceExport validation", "memberCluster", mcObjectName) + return admission.Allowed("Networking agents disabled; skipping ServiceExport validation") } klog.V(2).InfoS("Validating webhook member cluster DELETE", "memberCluster", mcObjectName) diff --git a/pkg/webhook/membercluster/membercluster_validating_webhook_test.go b/pkg/webhook/membercluster/membercluster_validating_webhook_test.go new file mode 100644 index 000000000..7cc231e41 --- /dev/null +++ b/pkg/webhook/membercluster/membercluster_validating_webhook_test.go @@ -0,0 +1,141 @@ +package membercluster + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + + clusterv1beta1 "github.com/kubefleet-dev/kubefleet/apis/cluster/v1beta1" + "github.com/kubefleet-dev/kubefleet/pkg/utils" + + fleetnetworkingv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func TestHandleDelete(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + networkingEnabled bool + validationMode clusterv1beta1.DeleteValidationMode + wantAllowed bool + wantMessageSubstr string + }{ + "networking-disabled-allows-delete": { + networkingEnabled: false, + wantAllowed: true, + validationMode: clusterv1beta1.DeleteValidationModeStrict, + }, + "networking-enabled-denies-delete": { + networkingEnabled: true, + wantAllowed: false, + validationMode: clusterv1beta1.DeleteValidationModeStrict, + wantMessageSubstr: "Please delete serviceExport", + }, + "delete-options-skip-bypasses-validation": { + networkingEnabled: true, + wantAllowed: true, + validationMode: clusterv1beta1.DeleteValidationModeSkip, + }, + } + + for name, tc := range testCases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + mcName := fmt.Sprintf("member-%s", name) + namespaceName := fmt.Sprintf(utils.NamespaceNameFormat, mcName) + svcExport := newInternalServiceExport(mcName, namespaceName) + + validator := newMemberClusterValidatorForTest(t, tc.networkingEnabled, svcExport) + mc := &clusterv1beta1.MemberCluster{ObjectMeta: metav1.ObjectMeta{Name: mcName}} + mc.Spec.DeleteOptions = &clusterv1beta1.DeleteOptions{ValidationMode: tc.validationMode} + req := buildDeleteRequestFromObject(t, mc) + + resp := validator.Handle(context.Background(), req) + if resp.Allowed != tc.wantAllowed { + t.Fatalf("Handle() got response: %+v, want allowed %t", resp, tc.wantAllowed) + } + if tc.wantMessageSubstr != "" { + if resp.Result == nil || !strings.Contains(resp.Result.Message, tc.wantMessageSubstr) { + t.Fatalf("Handle() got response result: %v, want contain: %q", resp.Result, tc.wantMessageSubstr) + } + } + }) + } +} + +func newMemberClusterValidatorForTest(t *testing.T, networkingEnabled bool, objs ...client.Object) *memberClusterValidator { + t.Helper() + + scheme := runtime.NewScheme() + if err := clusterv1beta1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add member cluster scheme: %v", err) + } + if err := fleetnetworkingv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add fleet networking scheme: %v", err) + } + scheme.AddKnownTypes(fleetnetworkingv1alpha1.GroupVersion, + &fleetnetworkingv1alpha1.InternalServiceExport{}, + &fleetnetworkingv1alpha1.InternalServiceExportList{}, + ) + metav1.AddToGroupVersion(scheme, fleetnetworkingv1alpha1.GroupVersion) + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() + decoder := admission.NewDecoder(scheme) + + return &memberClusterValidator{ + client: fakeClient, + decoder: decoder, + networkingAgentsEnabled: networkingEnabled, + } +} + +func buildDeleteRequestFromObject(t *testing.T, mc *clusterv1beta1.MemberCluster) admission.Request { + t.Helper() + + raw, err := json.Marshal(mc) + if err != nil { + t.Fatalf("failed to marshal member cluster: %v", err) + } + + return admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + Name: mc.Name, + OldObject: runtime.RawExtension{Raw: raw}, + }, + } +} + +func newInternalServiceExport(clusterID, namespace string) *fleetnetworkingv1alpha1.InternalServiceExport { + return &fleetnetworkingv1alpha1.InternalServiceExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-service", + Namespace: namespace, + }, + Spec: fleetnetworkingv1alpha1.InternalServiceExportSpec{ + ServiceReference: fleetnetworkingv1alpha1.ExportedObjectReference{ + ClusterID: clusterID, + Kind: "Service", + Namespace: "work", + Name: "sample-service", + ResourceVersion: "1", + Generation: 1, + UID: types.UID("svc-uid"), + NamespacedName: "work/sample-service", + }, + }, + } +} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 781833e3e..4f4c89b56 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -131,14 +131,16 @@ var ( var AddToManagerFuncs []func(manager.Manager) error var AddToManagerFleetResourceValidator func(manager.Manager, []string, bool) error +var AddToManagerMemberclusterValidator func(manager.Manager, bool) // AddToManager adds all Controllers to the Manager -func AddToManager(m manager.Manager, whiteListedUsers []string, denyModifyMemberClusterLabels bool) error { +func AddToManager(m manager.Manager, whiteListedUsers []string, denyModifyMemberClusterLabels bool, networkingAgentsEnabled bool) error { for _, f := range AddToManagerFuncs { if err := f(m); err != nil { return err } } + AddToManagerMemberclusterValidator(m, networkingAgentsEnabled) return AddToManagerFleetResourceValidator(m, whiteListedUsers, denyModifyMemberClusterLabels) } diff --git a/test/e2e/join_and_leave_test.go b/test/e2e/join_and_leave_test.go index 5b6e79f0c..d8a692809 100644 --- a/test/e2e/join_and_leave_test.go +++ b/test/e2e/join_and_leave_test.go @@ -17,9 +17,7 @@ limitations under the License. package e2e import ( - "errors" "fmt" - "reflect" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -163,34 +161,8 @@ var _ = Describe("Test member cluster join and leave flow", Label("joinleave"), } }) - It("Should fail the unjoin requests", func() { - for idx := range allMemberClusters { - memberCluster := allMemberClusters[idx] - mcObj := &clusterv1beta1.MemberCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: memberCluster.ClusterName, - }, - } - err := hubClient.Delete(ctx, mcObj) - Expect(err).ShouldNot(Succeed(), "Want the deletion to be denied") - var statusErr *apierrors.StatusError - Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Delete memberCluster call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&apierrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("Please delete serviceExport test-namespace/test-svc in the member cluster before leaving, request is denied")) - } - }) - - It("Deleting the internalServiceExports", func() { - for idx := range allMemberClusterNames { - memberCluster := allMemberClusters[idx] - namespaceName := fmt.Sprintf(utils.NamespaceNameFormat, memberCluster.ClusterName) - - internalSvcExportKey := types.NamespacedName{Namespace: namespaceName, Name: internalServiceExportName} - var export fleetnetworkingv1alpha1.InternalServiceExport - Expect(hubClient.Get(ctx, internalSvcExportKey, &export)).Should(Succeed(), "Failed to get internalServiceExport") - Expect(hubClient.Delete(ctx, &export)).To(Succeed(), "Failed to delete internalServiceExport") - } - }) - + // The network agent is not turned on by default in the e2e so we are still able to leave when we have internalServiceExport + // TODO: add a test case for the network agent is turned on in the fleet network repository. It("Should be able to trigger the member cluster DELETE", func() { setAllMemberClustersToLeave() }) @@ -362,22 +334,7 @@ var _ = Describe("Test member cluster join and leave flow", Label("joinleave"), } }) - It("Should fail the unjoin requests", func() { - for idx := range allMemberClusters { - memberCluster := allMemberClusters[idx] - mcObj := &clusterv1beta1.MemberCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: memberCluster.ClusterName, - }, - } - err := hubClient.Delete(ctx, mcObj) - Expect(err).ShouldNot(Succeed(), "Want the deletion to be denied") - var statusErr *apierrors.StatusError - Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Delete memberCluster call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&apierrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("Please delete serviceExport test-namespace/test-svc in the member cluster before leaving, request is denied")) - } - }) - + // It does not really matter here as the network agent is not turned on by default in the e2e so we are still able to leave when we have internalServiceExport It("Updating the member cluster to skip validation", func() { for idx := range allMemberClusterNames { memberCluster := allMemberClusters[idx] From 6862d17d087494fe283b8264b6a7da529b573fda Mon Sep 17 00:00:00 2001 From: Ryan Zhang Date: Mon, 8 Dec 2025 12:58:32 -0800 Subject: [PATCH 15/16] fix: continue to propagate PVC when workload is not enabled (#368) --- cmd/hubagent/workload/setup.go | 2 + pkg/controllers/placement/controller.go | 3 + .../placement/resource_selector.go | 2 +- pkg/resourcewatcher/change_dector.go | 5 +- pkg/utils/common.go | 24 ++++--- pkg/utils/common_test.go | 72 +++++++++++++++++-- 6 files changed, 89 insertions(+), 19 deletions(-) diff --git a/cmd/hubagent/workload/setup.go b/cmd/hubagent/workload/setup.go index cbaee55d5..dbb47de4f 100644 --- a/cmd/hubagent/workload/setup.go +++ b/cmd/hubagent/workload/setup.go @@ -168,6 +168,7 @@ func SetupControllers(ctx context.Context, wg *sync.WaitGroup, mgr ctrl.Manager, UncachedReader: mgr.GetAPIReader(), ResourceSnapshotCreationMinimumInterval: opts.ResourceSnapshotCreationMinimumInterval, ResourceChangesCollectionDuration: opts.ResourceChangesCollectionDuration, + EnableWorkload: opts.EnableWorkload, } rateLimiter := options.DefaultControllerRateLimiter(opts.RateLimiterOpts) @@ -507,6 +508,7 @@ func SetupControllers(ctx context.Context, wg *sync.WaitGroup, mgr ctrl.Manager, SkippedNamespaces: skippedNamespaces, ConcurrentPlacementWorker: int(math.Ceil(float64(opts.MaxConcurrentClusterPlacement) / 10)), ConcurrentResourceChangeWorker: opts.ConcurrentResourceChangeSyncs, + EnableWorkload: opts.EnableWorkload, } if err := mgr.Add(resourceChangeDetector); err != nil { diff --git a/pkg/controllers/placement/controller.go b/pkg/controllers/placement/controller.go index 0e7351994..f366e689d 100644 --- a/pkg/controllers/placement/controller.go +++ b/pkg/controllers/placement/controller.go @@ -94,6 +94,9 @@ type Reconciler struct { // ResourceChangesCollectionDuration is the duration for collecting resource changes into one snapshot. ResourceChangesCollectionDuration time.Duration + + // EnableWorkload indicates whether workloads are allowed to run on the hub cluster. + EnableWorkload bool } func (r *Reconciler) Reconcile(ctx context.Context, key controller.QueueKey) (ctrl.Result, error) { diff --git a/pkg/controllers/placement/resource_selector.go b/pkg/controllers/placement/resource_selector.go index 51a4d995a..0805889b7 100644 --- a/pkg/controllers/placement/resource_selector.go +++ b/pkg/controllers/placement/resource_selector.go @@ -294,7 +294,7 @@ func (r *Reconciler) shouldPropagateObj(namespace, placementName string, obj run return false, nil } - shouldInclude, err := utils.ShouldPropagateObj(r.InformerManager, uObj) + shouldInclude, err := utils.ShouldPropagateObj(r.InformerManager, uObj, r.EnableWorkload) if err != nil { klog.ErrorS(err, "Cannot determine if we should propagate an object", "namespace", namespace, "placement", placementName, "object", uObjKObj) return false, err diff --git a/pkg/resourcewatcher/change_dector.go b/pkg/resourcewatcher/change_dector.go index 41760a185..9a4011628 100644 --- a/pkg/resourcewatcher/change_dector.go +++ b/pkg/resourcewatcher/change_dector.go @@ -84,6 +84,9 @@ type ChangeDetector struct { // ConcurrentResourceChangeWorker is the number of resource change work that are // allowed to sync concurrently. ConcurrentResourceChangeWorker int + + // EnableWorkload indicates whether workloads are allowed to run on the hub cluster. + EnableWorkload bool } // Start runs the detector, never stop until stopCh closed. This is called by the controller manager. @@ -189,7 +192,7 @@ func (d *ChangeDetector) dynamicResourceFilter(obj interface{}) bool { } if unstructuredObj, ok := obj.(*unstructured.Unstructured); ok { - shouldPropagate, err := utils.ShouldPropagateObj(d.InformerManager, unstructuredObj.DeepCopy()) + shouldPropagate, err := utils.ShouldPropagateObj(d.InformerManager, unstructuredObj.DeepCopy(), d.EnableWorkload) if err != nil || !shouldPropagate { klog.V(5).InfoS("Skip watching resource in namespace", "namespace", cwKey.Namespace, "group", cwKey.Group, "version", cwKey.Version, "kind", cwKey.Kind, "object", cwKey.Name) diff --git a/pkg/utils/common.go b/pkg/utils/common.go index 63225001b..03b0589df 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -505,29 +505,30 @@ func CheckCRDInstalled(discoveryClient discovery.DiscoveryInterface, gvk schema. return err } -// ShouldPropagateObj decides if one should propagate the object -func ShouldPropagateObj(informerManager informer.Manager, uObj *unstructured.Unstructured) (bool, error) { +// ShouldPropagateObj decides if one should propagate the object. +// PVCs are only propagated when enableWorkload is false (workloads not allowed on hub). +func ShouldPropagateObj(informerManager informer.Manager, uObj *unstructured.Unstructured, enableWorkload bool) (bool, error) { // TODO: add more special handling for different resource kind switch uObj.GroupVersionKind() { case appv1.SchemeGroupVersion.WithKind(ReplicaSetKind): - // Skip ReplicaSets if they are managed by Deployments (have owner references) - // Standalone ReplicaSets (without owners) can be propagated + // Skip ReplicaSets if they are managed by Deployments (have owner references). + // Standalone ReplicaSets (without owners) can be propagated. if len(uObj.GetOwnerReferences()) > 0 { return false, nil } case appv1.SchemeGroupVersion.WithKind("ControllerRevision"): - // Skip ControllerRevisions if they are managed by DaemonSets/StatefulSets (have owner references) - // These are automatically created by controllers and will be recreated on member clusters + // Skip ControllerRevisions if they are managed by DaemonSets/StatefulSets (have owner references). + // Standalone ControllerRevisions (without owners) can be propagated. if len(uObj.GetOwnerReferences()) > 0 { return false, nil } case corev1.SchemeGroupVersion.WithKind(ConfigMapKind): - // Skip the built-in custom CA certificate created in the namespace + // Skip the built-in custom CA certificate created in the namespace. if uObj.GetName() == "kube-root-ca.crt" { return false, nil } case corev1.SchemeGroupVersion.WithKind("ServiceAccount"): - // Skip the default service account created in the namespace + // Skip the default service account created in the namespace. if uObj.GetName() == "default" { return false, nil } @@ -542,8 +543,11 @@ func ShouldPropagateObj(informerManager informer.Manager, uObj *unstructured.Uns return false, nil } case corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim"): - // Skip PersistentVolumeClaims to avoid conflicts with the PVCs created by statefulset controller - return false, nil + // Skip PersistentVolumeClaims by default to avoid conflicts with the PVCs created by statefulset controller. + // This only happens if the workloads are allowed to run on the hub cluster. + if enableWorkload { + return false, nil + } case corev1.SchemeGroupVersion.WithKind("Endpoints"): // we assume that all endpoints with the same name of a service is created by the service controller if _, err := informerManager.Lister(ServiceGVR).ByNamespace(uObj.GetNamespace()).Get(uObj.GetName()); err != nil { diff --git a/pkg/utils/common_test.go b/pkg/utils/common_test.go index e6f9433d3..d99d11711 100644 --- a/pkg/utils/common_test.go +++ b/pkg/utils/common_test.go @@ -1191,11 +1191,12 @@ func TestIsDiffedResourcePlacementEqual(t *testing.T) { } } -func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { +func TestShouldPropagateObj(t *testing.T) { tests := []struct { name string obj map[string]interface{} ownerReferences []metav1.OwnerReference + enableWorkload bool want bool }{ { @@ -1209,6 +1210,21 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { }, }, ownerReferences: nil, + enableWorkload: true, + want: true, + }, + { + name: "standalone replicaset without ownerReferences should propagate if workload is disabled", + obj: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "metadata": map[string]interface{}{ + "name": "standalone-rs", + "namespace": "default", + }, + }, + ownerReferences: nil, + enableWorkload: false, want: true, }, { @@ -1222,6 +1238,7 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { }, }, ownerReferences: nil, + enableWorkload: true, want: true, }, { @@ -1242,7 +1259,8 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { UID: "12345", }, }, - want: false, + enableWorkload: true, + want: false, }, { name: "pod owned by replicaset - passes ShouldPropagateObj but filtered by resource config", @@ -1262,7 +1280,8 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { UID: "67890", }, }, - want: true, // ShouldPropagateObj doesn't filter Pods - they're filtered by NewResourceConfig + enableWorkload: false, + want: true, // ShouldPropagateObj doesn't filter Pods - they're filtered by NewResourceConfig }, { name: "controllerrevision owned by daemonset should NOT propagate", @@ -1282,7 +1301,8 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { UID: "abcdef", }, }, - want: false, + enableWorkload: false, + want: false, }, { name: "controllerrevision owned by statefulset should NOT propagate", @@ -1302,7 +1322,8 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { UID: "fedcba", }, }, - want: false, + enableWorkload: false, + want: false, }, { name: "standalone controllerrevision without owner should propagate", @@ -1315,10 +1336,25 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { }, }, ownerReferences: nil, + enableWorkload: false, + want: true, + }, + { + name: "PVC should propagate when workload is disabled", + obj: map[string]interface{}{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": map[string]interface{}{ + "name": "test-pvc", + "namespace": "default", + }, + }, + ownerReferences: nil, + enableWorkload: false, want: true, }, { - name: "PersistentVolumeClaim should NOT propagate", + name: "PVC should NOT propagate when workload is enabled", obj: map[string]interface{}{ "apiVersion": "v1", "kind": "PersistentVolumeClaim", @@ -1328,8 +1364,30 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { }, }, ownerReferences: nil, + enableWorkload: true, want: false, }, + { + name: "PVC with ownerReferences should NOT propagate when workload is enabled", + obj: map[string]interface{}{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": map[string]interface{}{ + "name": "data-statefulset-0", + "namespace": "default", + }, + }, + ownerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "statefulset", + UID: "sts-uid", + }, + }, + enableWorkload: true, + want: false, + }, } for _, tt := range tests { @@ -1339,7 +1397,7 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { uObj.SetOwnerReferences(tt.ownerReferences) } - got, err := ShouldPropagateObj(nil, uObj) + got, err := ShouldPropagateObj(nil, uObj, tt.enableWorkload) if err != nil { t.Errorf("ShouldPropagateObj() error = %v", err) return From 5730600956b450fc9dd9df522d646f16f887d3ce Mon Sep 17 00:00:00 2001 From: Wei Weng Date: Mon, 8 Dec 2025 23:08:43 +0000 Subject: [PATCH 16/16] Revert "feat: publish image to ghcr with tag (#353)" This reverts commit d5fcf88ff6e2728deb6350cdf4559bb6c3714954. Signed-off-by: Wei Weng --- .github/workflows/release.yml | 87 ----------------------------------- 1 file changed, 87 deletions(-) delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index d6fff39de..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: Release Images - -on: - push: - tags: - - 'v*.*.*' - workflow_dispatch: - inputs: - tag: - description: 'Release tag (e.g., v1.0.0)' - required: true - type: string - -permissions: - contents: read - packages: write - -env: - REGISTRY: ghcr.io - HUB_AGENT_IMAGE_NAME: hub-agent - MEMBER_AGENT_IMAGE_NAME: member-agent - REFRESH_TOKEN_IMAGE_NAME: refresh-token - GO_VERSION: '1.24.9' - -jobs: - export-registry: - runs-on: ubuntu-latest - outputs: - registry: ${{ steps.export.outputs.registry }} - tag: ${{ steps.export.outputs.tag }} - steps: - - name: Checkout code - uses: actions/checkout@v6.0.1 - - - id: export - run: | - # registry must be in lowercase - echo "registry=$(echo "${{ env.REGISTRY }}/${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT - - # Extract tag from github ref or workflow input - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - TAG="${{ inputs.tag }}" - elif [[ "${{ github.ref }}" == refs/tags/* ]]; then - TAG=${GITHUB_REF#refs/tags/} - else - echo "Error: Workflow triggered by unsupported event or ref" - echo "Event: ${{ github.event_name }}" - echo "Ref: ${{ github.ref }}" - exit 1 - fi - echo "tag=${TAG}" >> $GITHUB_OUTPUT - echo "Release tag: ${TAG}" - - build-and-publish: - needs: export-registry - env: - REGISTRY: ${{ needs.export-registry.outputs.registry }} - TAG: ${{ needs.export-registry.outputs.tag }} - runs-on: ubuntu-latest - steps: - - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v6 - with: - go-version: ${{ env.GO_VERSION }} - - - name: Checkout code - uses: actions/checkout@v6.0.1 - - - name: Login to ghcr.io - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push images with tag ${{ env.TAG }} - run: | - make push - - - name: Verify images - run: | - echo "✅ Published images:" - echo " - ${{ env.REGISTRY }}/${{ env.HUB_AGENT_IMAGE_NAME }}:${{ env.TAG }}" - echo " - ${{ env.REGISTRY }}/${{ env.MEMBER_AGENT_IMAGE_NAME }}:${{ env.TAG }}" - echo " - ${{ env.REGISTRY }}/${{ env.REFRESH_TOKEN_IMAGE_NAME }}:${{ env.TAG }}" - echo "" - echo "📦 Images are now public!"