diff --git a/.github/workflows/chart.yml b/.github/workflows/chart.yml index f1d2a197c..a2d709555 100644 --- a/.github/workflows/chart.yml +++ b/.github/workflows/chart.yml @@ -18,7 +18,7 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 with: submodules: true fetch-depth: 0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9a11c656..9c999af35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 - name: Set up Ginkgo CLI run: | @@ -90,7 +90,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 - name: Move Docker data directory to /mnt # The default storage device on GitHub-hosted runners is running low during e2e tests. diff --git a/.github/workflows/code-lint.yml b/.github/workflows/code-lint.yml index a49324aed..ac3906219 100644 --- a/.github/workflows/code-lint.yml +++ b/.github/workflows/code-lint.yml @@ -43,7 +43,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Checkout - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 with: submodules: true @@ -64,7 +64,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 - name: golangci-lint run: make lint diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2536446da..91deab101 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index c91f5d2d8..5730dfc94 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -12,11 +12,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 + uses: step-security/harden-runner@df199fb7be9f65074067a9eb93f12bb4c5547cf2 # v2.13.3 with: egress-policy: audit - - uses: actions/checkout@c2d88d3ecc89a9ef08eebf45d9637801dcee7eb5 # v4.1.7 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v4.1.7 - uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # master with: check_filenames: true diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml index 7c8815f5a..bb2858063 100644 --- a/.github/workflows/markdown-lint.yml +++ b/.github/workflows/markdown-lint.yml @@ -10,7 +10,7 @@ jobs: markdown-link-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.0 + - uses: actions/checkout@v6.0.1 - uses: tcort/github-action-markdown-link-check@v1 with: # this will only show errors in the output diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index ab45f089a..995d8dd19 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -44,7 +44,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Checkout code - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 - name: Login to ${{ env.REGISTRY }} uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef diff --git a/.github/workflows/upgrade.yml b/.github/workflows/upgrade.yml index b4a697ec9..1bfaff9df 100644 --- a/.github/workflows/upgrade.yml +++ b/.github/workflows/upgrade.yml @@ -44,7 +44,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 with: # Fetch the history of all branches and tags. # This is needed for the test suite to switch between releases. @@ -146,7 +146,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 with: # Fetch the history of all branches and tags. # This is needed for the test suite to switch between releases. @@ -248,7 +248,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: Check out code into the Go module directory - uses: actions/checkout@v6.0.0 + uses: actions/checkout@v6.0.1 with: # Fetch the history of all branches and tags. # This is needed for the test suite to switch between releases. diff --git a/apis/placement/v1beta1/commons.go b/apis/placement/v1beta1/commons.go index 03e8e9dc0..3d118dff0 100644 --- a/apis/placement/v1beta1/commons.go +++ b/apis/placement/v1beta1/commons.go @@ -156,7 +156,10 @@ const ( UpdateRunFinalizer = FleetPrefix + "stagedupdaterun-finalizer" // TargetUpdateRunLabel indicates the target update run on a staged run related object. - TargetUpdateRunLabel = FleetPrefix + "targetupdaterun" + TargetUpdateRunLabel = FleetPrefix + "targetUpdateRun" + + // TaskTypeLabel indicates the task type (before-stage or after-stage) on a staged run related object. + TaskTypeLabel = FleetPrefix + "taskType" // UpdateRunDeleteStageName is the name of delete stage in the staged update run. UpdateRunDeleteStageName = FleetPrefix + "deleteStage" @@ -167,8 +170,17 @@ const ( // TargetUpdatingStageNameLabel indicates the updating stage name on a staged run related object. TargetUpdatingStageNameLabel = FleetPrefix + "targetUpdatingStage" - // ApprovalTaskNameFmt is the format of the approval task name. - ApprovalTaskNameFmt = "%s-%s" + // BeforeStageTaskLabelValue is the before stage task label value. + BeforeStageTaskLabelValue = "beforeStage" + + // AfterStageTaskLabelValue is the after stage task label value. + AfterStageTaskLabelValue = "afterStage" + + // BeforeStageApprovalTaskNameFmt is the format of the before stage approval task name. + BeforeStageApprovalTaskNameFmt = "%s-before-%s" + + // AfterStageApprovalTaskNameFmt is the format of the after stage approval task name. + AfterStageApprovalTaskNameFmt = "%s-after-%s" ) var ( diff --git a/apis/placement/v1beta1/stageupdate_types.go b/apis/placement/v1beta1/stageupdate_types.go index 1144d02ea..8d41589a7 100644 --- a/apis/placement/v1beta1/stageupdate_types.go +++ b/apis/placement/v1beta1/stageupdate_types.go @@ -152,29 +152,25 @@ func (c *ClusterStagedUpdateRun) SetUpdateRunStatus(status UpdateRunStatus) { type State string const ( - // StateNotStarted describes user intent to initialize but not execute the update run. + // StateInitialize describes user intent to initialize but not run the update run. // This is the default state when an update run is created. - StateNotStarted State = "Initialize" + // Users can subsequently set the state to Run. + StateInitialize State = "Initialize" - // StateStarted describes user intent to execute (or resume execution if paused). - // Users can subsequently set the state to Pause or Abandon. - StateStarted State = "Execute" + // StateRun describes user intent to execute (or resume execution if stopped). + // Users can subsequently set the state to Stop. + StateRun State = "Run" - // StateStopped describes user intent to pause the update run. - // Users can subsequently set the state to Execute or Abandon. - StateStopped State = "Pause" - - // StateAbandoned describes user intent to abandon the update run. - // This is a terminal state; once set, it cannot be changed. - StateAbandoned State = "Abandon" + // StateStop describes user intent to stop the update run. + // Users can subsequently set the state to Run. + StateStop State = "Stop" ) // UpdateRunSpec defines the desired rollout strategy and the snapshot indices of the resources to be updated. // It specifies a stage-by-stage update process across selected clusters for the given ResourcePlacement object. -// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Initialize' && self.state == 'Pause')",message="invalid state transition: cannot transition from Initialize to Pause" -// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Execute' && self.state == 'Initialize')",message="invalid state transition: cannot transition from Execute to Initialize" -// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Pause' && self.state == 'Initialize')",message="invalid state transition: cannot transition from Pause to Initialize" -// +kubebuilder:validation:XValidation:rule="!has(oldSelf.state) || oldSelf.state != 'Abandon' || self.state == 'Abandon'",message="invalid state transition: Abandon is a terminal state and cannot transition to any other state" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Initialize' && self.state == 'Stop')",message="invalid state transition: cannot transition from Initialize to Stop" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Run' && self.state == 'Initialize')",message="invalid state transition: cannot transition from Run to Initialize" +// +kubebuilder:validation:XValidation:rule="!(has(oldSelf.state) && oldSelf.state == 'Stop' && self.state == 'Initialize')",message="invalid state transition: cannot transition from Stop to Initialize" type UpdateRunSpec struct { // PlacementName is the name of placement that this update run is applied to. // There can be multiple active update runs for each placement, but @@ -200,12 +196,11 @@ type UpdateRunSpec struct { // State indicates the desired state of the update run. // Initialize: The update run should be initialized but execution should not start (default). - // Execute: The update run should execute or resume execution. - // Pause: The update run should pause execution. - // Abandon: The update run should be abandoned and terminated. + // Run: The update run should execute or resume execution. + // Stop: The update run should stop execution. // +kubebuilder:validation:Optional // +kubebuilder:default=Initialize - // +kubebuilder:validation:Enum=Initialize;Execute;Pause;Abandon + // +kubebuilder:validation:Enum=Initialize;Run;Stop State State `json:"state,omitempty"` } @@ -426,7 +421,6 @@ const ( // Its condition status can be one of the following: // - "True": The staged update run is initialized successfully. // - "False": The staged update run encountered an error during initialization and aborted. - // - "Unknown": The staged update run initialization has started. StagedUpdateRunConditionInitialized StagedUpdateRunConditionType = "Initialized" // StagedUpdateRunConditionProgressing indicates whether the staged update run is making progress. @@ -679,6 +673,7 @@ type ApprovalRequestObjList interface { // - `TargetUpdateRun`: Points to the cluster staged update run that this approval request is for. // - `TargetStage`: The name of the stage that this approval request is for. // - `IsLatestUpdateRunApproval`: Indicates whether this approval request is the latest one related to this update run. +// - `TaskType`: Indicates whether this approval request is for the before or after stage task. type ClusterApprovalRequest struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -925,6 +920,7 @@ func (s *StagedUpdateStrategyList) GetUpdateStrategyObjs() []UpdateStrategyObj { // - `TargetUpdateRun`: Points to the staged update run that this approval request is for. // - `TargetStage`: The name of the stage that this approval request is for. // - `IsLatestUpdateRunApproval`: Indicates whether this approval request is the latest one related to this update run. +// - `TaskType`: Indicates whether this approval request is for the before or after stage task. type ApprovalRequest struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/apis/placement/v1beta1/work_types.go b/apis/placement/v1beta1/work_types.go index f4592f3d7..d2b04e6e3 100644 --- a/apis/placement/v1beta1/work_types.go +++ b/apis/placement/v1beta1/work_types.go @@ -54,6 +54,10 @@ const ( // WorkConditionTypeDiffReported reports whether Fleet has successfully reported the // configuration difference between the states in the hub cluster and a member cluster. WorkConditionTypeDiffReported = "DiffReported" + + // WorkConditionTypeStatusTrimmed reports whether the member agent has to trim + // the status data in the Work object due to size constraints. + WorkConditionTypeStatusTrimmed = "StatusTrimmed" ) // This api is copied from https://github.com/kubernetes-sigs/work-api/blob/master/pkg/apis/v1alpha1/work_types.go. diff --git a/cmd/hubagent/main.go b/cmd/hubagent/main.go index edd03fb5c..9d1608a3b 100644 --- a/cmd/hubagent/main.go +++ b/cmd/hubagent/main.go @@ -157,7 +157,8 @@ func main() { if opts.EnableWebhook { whiteListedUsers := strings.Split(opts.WhiteListedUsers, ",") - if err := SetupWebhook(mgr, options.WebhookClientConnectionType(opts.WebhookClientConnectionType), opts.WebhookServiceName, whiteListedUsers, opts.EnableGuardRail, opts.EnableV1Beta1APIs, opts.DenyModifyMemberClusterLabels, opts.EnableWorkload); err != nil { + if err := SetupWebhook(mgr, options.WebhookClientConnectionType(opts.WebhookClientConnectionType), opts.WebhookServiceName, whiteListedUsers, + opts.EnableGuardRail, opts.EnableV1Beta1APIs, opts.DenyModifyMemberClusterLabels, opts.EnableWorkload, opts.NetworkingAgentsEnabled); err != nil { klog.ErrorS(err, "unable to set up webhook") exitWithErrorFunc() } @@ -201,7 +202,8 @@ func main() { } // SetupWebhook generates the webhook cert and then set up the webhook configurator. -func SetupWebhook(mgr manager.Manager, webhookClientConnectionType options.WebhookClientConnectionType, webhookServiceName string, whiteListedUsers []string, enableGuardRail, isFleetV1Beta1API bool, denyModifyMemberClusterLabels bool, enableWorkload bool) error { +func SetupWebhook(mgr manager.Manager, webhookClientConnectionType options.WebhookClientConnectionType, webhookServiceName string, + whiteListedUsers []string, enableGuardRail, isFleetV1Beta1API bool, denyModifyMemberClusterLabels bool, enableWorkload bool, networkingAgentsEnabled bool) error { // Generate self-signed key and crt files in FleetWebhookCertDir for the webhook server to start. w, err := webhook.NewWebhookConfig(mgr, webhookServiceName, FleetWebhookPort, &webhookClientConnectionType, FleetWebhookCertDir, enableGuardRail, denyModifyMemberClusterLabels, enableWorkload) if err != nil { @@ -212,7 +214,7 @@ func SetupWebhook(mgr manager.Manager, webhookClientConnectionType options.Webho klog.ErrorS(err, "unable to add WebhookConfig") return err } - if err = webhook.AddToManager(mgr, whiteListedUsers, denyModifyMemberClusterLabels); err != nil { + if err = webhook.AddToManager(mgr, whiteListedUsers, denyModifyMemberClusterLabels, networkingAgentsEnabled); err != nil { klog.ErrorS(err, "unable to register webhooks to the manager") return err } diff --git a/cmd/hubagent/workload/setup.go b/cmd/hubagent/workload/setup.go index 748c40c3e..a2bccc07b 100644 --- a/cmd/hubagent/workload/setup.go +++ b/cmd/hubagent/workload/setup.go @@ -172,6 +172,7 @@ func SetupControllers(ctx context.Context, wg *sync.WaitGroup, mgr ctrl.Manager, UncachedReader: mgr.GetAPIReader(), ResourceSnapshotCreationMinimumInterval: opts.ResourceSnapshotCreationMinimumInterval, ResourceChangesCollectionDuration: opts.ResourceChangesCollectionDuration, + EnableWorkload: opts.EnableWorkload, } rateLimiter := options.DefaultControllerRateLimiter(opts.RateLimiterOpts) @@ -525,6 +526,7 @@ func SetupControllers(ctx context.Context, wg *sync.WaitGroup, mgr ctrl.Manager, SkippedNamespaces: skippedNamespaces, ConcurrentPlacementWorker: int(math.Ceil(float64(opts.MaxConcurrentClusterPlacement) / 10)), ConcurrentResourceChangeWorker: opts.ConcurrentResourceChangeSyncs, + EnableWorkload: opts.EnableWorkload, } if err := mgr.Add(resourceChangeDetector); err != nil { diff --git a/config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml b/config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml index 3835fd87c..b16f6ea71 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml @@ -41,6 +41,7 @@ spec: - `TargetUpdateRun`: Points to the staged update run that this approval request is for. - `TargetStage`: The name of the stage that this approval request is for. - `IsLatestUpdateRunApproval`: Indicates whether this approval request is the latest one related to this update run. + - `TaskType`: Indicates whether this approval request is for the before or after stage task. properties: apiVersion: description: |- diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterapprovalrequests.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterapprovalrequests.yaml index 56f4b7552..2333e23f1 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterapprovalrequests.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterapprovalrequests.yaml @@ -172,6 +172,7 @@ spec: - `TargetUpdateRun`: Points to the cluster staged update run that this approval request is for. - `TargetStage`: The name of the stage that this approval request is for. - `IsLatestUpdateRunApproval`: Indicates whether this approval request is the latest one related to this update run. + - `TaskType`: Indicates whether this approval request is for the before or after stage task. properties: apiVersion: description: |- diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml index c95748724..725eb8ddc 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml @@ -1189,14 +1189,12 @@ spec: description: |- State indicates the desired state of the update run. Initialize: The update run should be initialized but execution should not start (default). - Execute: The update run should execute or resume execution. - Pause: The update run should pause execution. - Abandon: The update run should be abandoned and terminated. + Run: The update run should execute or resume execution. + Stop: The update run should stop execution. enum: - Initialize - - Execute - - Pause - - Abandon + - Run + - Stop type: string required: - placementName @@ -1204,21 +1202,15 @@ spec: type: object x-kubernetes-validations: - message: 'invalid state transition: cannot transition from Initialize - to Pause' + to Stop' rule: '!(has(oldSelf.state) && oldSelf.state == ''Initialize'' && self.state - == ''Pause'')' - - message: 'invalid state transition: cannot transition from Execute to - Initialize' - rule: '!(has(oldSelf.state) && oldSelf.state == ''Execute'' && self.state + == ''Stop'')' + - message: 'invalid state transition: cannot transition from Run to Initialize' + rule: '!(has(oldSelf.state) && oldSelf.state == ''Run'' && self.state == ''Initialize'')' - - message: 'invalid state transition: cannot transition from Pause to - Initialize' - rule: '!(has(oldSelf.state) && oldSelf.state == ''Pause'' && self.state + - message: 'invalid state transition: cannot transition from Stop to Initialize' + rule: '!(has(oldSelf.state) && oldSelf.state == ''Stop'' && self.state == ''Initialize'')' - - message: 'invalid state transition: Abandon is a terminal state and - cannot transition to any other state' - rule: '!has(oldSelf.state) || oldSelf.state != ''Abandon'' || self.state - == ''Abandon''' status: description: The observed status of ClusterStagedUpdateRun. properties: diff --git a/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml index abfa39f46..b06ff9829 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml @@ -109,14 +109,12 @@ spec: description: |- State indicates the desired state of the update run. Initialize: The update run should be initialized but execution should not start (default). - Execute: The update run should execute or resume execution. - Pause: The update run should pause execution. - Abandon: The update run should be abandoned and terminated. + Run: The update run should execute or resume execution. + Stop: The update run should stop execution. enum: - Initialize - - Execute - - Pause - - Abandon + - Run + - Stop type: string required: - placementName @@ -124,21 +122,15 @@ spec: type: object x-kubernetes-validations: - message: 'invalid state transition: cannot transition from Initialize - to Pause' + to Stop' rule: '!(has(oldSelf.state) && oldSelf.state == ''Initialize'' && self.state - == ''Pause'')' - - message: 'invalid state transition: cannot transition from Execute to - Initialize' - rule: '!(has(oldSelf.state) && oldSelf.state == ''Execute'' && self.state + == ''Stop'')' + - message: 'invalid state transition: cannot transition from Run to Initialize' + rule: '!(has(oldSelf.state) && oldSelf.state == ''Run'' && self.state == ''Initialize'')' - - message: 'invalid state transition: cannot transition from Pause to - Initialize' - rule: '!(has(oldSelf.state) && oldSelf.state == ''Pause'' && self.state + - message: 'invalid state transition: cannot transition from Stop to Initialize' + rule: '!(has(oldSelf.state) && oldSelf.state == ''Stop'' && self.state == ''Initialize'')' - - message: 'invalid state transition: Abandon is a terminal state and - cannot transition to any other state' - rule: '!has(oldSelf.state) || oldSelf.state != ''Abandon'' || self.state - == ''Abandon''' status: description: The observed status of StagedUpdateRun. properties: diff --git a/go.mod b/go.mod index edd81759c..c8524cc40 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 github.com/Azure/karpenter-provider-azure v1.5.1 - github.com/crossplane/crossplane-runtime v1.20.0 + github.com/crossplane/crossplane-runtime/v2 v2.1.0 github.com/evanphx/json-patch/v5 v5.9.11 github.com/gofrs/uuid v4.4.0+incompatible github.com/google/go-cmp v0.7.0 @@ -41,11 +41,11 @@ require ( sigs.k8s.io/cloud-provider-azure v1.32.4 sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.5.20 sigs.k8s.io/cluster-inventory-api v0.0.0-20251028164203-2e3fabb46733 - sigs.k8s.io/controller-runtime v0.21.0 + sigs.k8s.io/controller-runtime v0.22.4 ) require ( - dario.cat/mergo v1.0.1 // indirect + dario.cat/mergo v1.0.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.4.0 // indirect diff --git a/go.sum b/go.sum index c8c2987a0..7e51f3fd6 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= -dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/Azure/aks-middleware v0.0.40 h1:eFRuAxCcIAZoy/6+FvumDl2KOWnSPxXcAeCSOA4+aTo= github.com/Azure/aks-middleware v0.0.40/go.mod h1:7Y+wxZmS7p1K0FPreiO3+6Wr8YhYjWz9c50YohDQIQ4= github.com/Azure/azure-kusto-go v0.16.1 h1:vCBWcQghmC1qIErUUgVNWHxGhZVStu1U/hki6iBA14k= @@ -97,8 +97,8 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/crossplane/crossplane-runtime v1.20.0 h1:I54uipRIecqZyms+vz1J/l62yjVQ7HV5w+Nh3RMrUtc= -github.com/crossplane/crossplane-runtime v1.20.0/go.mod h1:lfV1VJenDc9PNVLxDC80YjPoTm+JdSZ13xlS2h37Dvg= +github.com/crossplane/crossplane-runtime/v2 v2.1.0 h1:JBMhL9T+/PfyjLAQEdZWlKLvA3jJVtza8zLLwd9Gs4k= +github.com/crossplane/crossplane-runtime/v2 v2.1.0/go.mod h1:j78pmk0qlI//Ur7zHhqTr8iePHFcwJKrZnzZB+Fg4t0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -435,8 +435,8 @@ sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.5.2 h1:jjFJF0PmS9I sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.5.2/go.mod h1:7DdZ9ipIsmPLpBlfT4gueejcUlJBZQKWhdljQE5SKvc= sigs.k8s.io/cluster-inventory-api v0.0.0-20251028164203-2e3fabb46733 h1:l90ANqblqFrE4L2QLLk+9iPjfmaLRvOFL51l/fgwUgg= sigs.k8s.io/cluster-inventory-api v0.0.0-20251028164203-2e3fabb46733/go.mod h1:guwenlZ9iIfYlNxn7ExCfugOLTh6wjjRX3adC36YCmQ= -sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= -sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= +sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= +sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/karpenter v1.5.0 h1:3HaFtFvkteUJ+SjIViR1ImR0qR+GTqDulahauIuE4Qg= diff --git a/pkg/controllers/internalmembercluster/v1beta1/member_controller_test.go b/pkg/controllers/internalmembercluster/v1beta1/member_controller_test.go index 041cd0ff0..15f9c22c7 100644 --- a/pkg/controllers/internalmembercluster/v1beta1/member_controller_test.go +++ b/pkg/controllers/internalmembercluster/v1beta1/member_controller_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/crossplane-runtime/v2/pkg/test" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" diff --git a/pkg/controllers/membercluster/v1beta1/membercluster_controller_test.go b/pkg/controllers/membercluster/v1beta1/membercluster_controller_test.go index a80e79127..087684a1a 100644 --- a/pkg/controllers/membercluster/v1beta1/membercluster_controller_test.go +++ b/pkg/controllers/membercluster/v1beta1/membercluster_controller_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/crossplane-runtime/v2/pkg/test" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/assert" diff --git a/pkg/controllers/placement/controller.go b/pkg/controllers/placement/controller.go index 4f88520b0..5b9bb6930 100644 --- a/pkg/controllers/placement/controller.go +++ b/pkg/controllers/placement/controller.go @@ -94,6 +94,9 @@ type Reconciler struct { // ResourceChangesCollectionDuration is the duration for collecting resource changes into one snapshot. ResourceChangesCollectionDuration time.Duration + + // EnableWorkload indicates whether workloads are allowed to run on the hub cluster. + EnableWorkload bool } func (r *Reconciler) Reconcile(ctx context.Context, key controller.QueueKey) (ctrl.Result, error) { diff --git a/pkg/controllers/placement/resource_selector.go b/pkg/controllers/placement/resource_selector.go index 801ca94a6..08d598536 100644 --- a/pkg/controllers/placement/resource_selector.go +++ b/pkg/controllers/placement/resource_selector.go @@ -294,7 +294,7 @@ func (r *Reconciler) shouldPropagateObj(namespace, placementName string, obj run return false, nil } - shouldInclude, err := utils.ShouldPropagateObj(r.InformerManager, uObj) + shouldInclude, err := utils.ShouldPropagateObj(r.InformerManager, uObj, r.EnableWorkload) if err != nil { klog.ErrorS(err, "Cannot determine if we should propagate an object", "namespace", namespace, "placement", placementName, "object", uObjKObj) return false, err diff --git a/pkg/controllers/rollout/controller_test.go b/pkg/controllers/rollout/controller_test.go index 5d2c9ca25..492ff81b1 100644 --- a/pkg/controllers/rollout/controller_test.go +++ b/pkg/controllers/rollout/controller_test.go @@ -1218,7 +1218,10 @@ func TestIsBindingReady(t *testing.T) { func TestPickBindingsToRoll(t *testing.T) { tests := map[string]struct { - allBindings []*placementv1beta1.ClusterResourceBinding + // We have to generate the bindings before calling PickBindingsToRoll instead of building them + // during the initialization. + // So that the waitTime is under the control. + allBindingsFunc func() []*placementv1beta1.ClusterResourceBinding latestResourceSnapshotName string crp *placementv1beta1.ClusterResourcePlacement matchedCROs []*placementv1beta1.ClusterResourceOverrideSnapshot @@ -1234,8 +1237,10 @@ func TestPickBindingsToRoll(t *testing.T) { }{ // TODO: add more tests "test scheduled binding to bound, outdated resources and nil overrides - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1253,8 +1258,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test scheduled binding to bound, outdated resources and updated apply strategy - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1276,8 +1283,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test scheduled binding to bound, outdated resources and empty overrides - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1297,8 +1306,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test scheduled binding to bound, outdated resources with overrides matching cluster - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1391,8 +1402,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test scheduled binding to bound, outdated resources with overrides not matching any cluster - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1474,8 +1487,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test scheduled binding to bound, overrides matching cluster - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -1568,8 +1583,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test bound binding with latest resources - rollout not needed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -1581,8 +1598,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantNeedRoll: false, }, "test failed to apply bound binding, outdated resources - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1600,12 +1619,14 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: defaultUnavailablePeriod * time.Second, }, "test one failed to apply bound binding and four failed non ready bound bindings, outdated resources with maxUnavailable specified - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1644,12 +1665,14 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: defaultUnavailablePeriod * time.Second, }, "test three failed to apply bound binding, two ready bound binding, outdated resources with maxUnavailable specified - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1688,12 +1711,14 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: defaultUnavailablePeriod * time.Second, }, "test bound ready bindings, maxUnavailable is set to zero - rollout blocked": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster4), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster5), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1742,7 +1767,9 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test with no bindings": { - allBindings: []*placementv1beta1.ClusterResourceBinding{}, + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{} + }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", createPlacementPolicyForTest(placementv1beta1.PickNPlacementType, 5), @@ -1752,9 +1779,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test two scheduled bindings, outdated resources - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster1), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1777,9 +1806,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test canBeReady bound and scheduled binding - rollout allowed with unavailable period wait time": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1813,9 +1844,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 60 * time.Second, }, "test two unscheduled bindings, maxUnavailable specified - rollout allowed": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -1831,8 +1864,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test overrides and the cluster is not found": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + } }, latestResourceSnapshotName: "snapshot-1", matchedCROs: []*placementv1beta1.ClusterResourceOverrideSnapshot{ @@ -1870,10 +1905,12 @@ func TestPickBindingsToRoll(t *testing.T) { }, "test bound bindings with different waitTimes and check the wait time should be the min of them all": { // want the min wait time of bound bindings that are not ready - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3, metav1.Time{Time: now.Add(-35 * time.Second)}), // notReady, waitTime = t - 35s - generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), // notReady, no wait time because it does not have available condition yet, - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-2", cluster2), // Ready + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster3, metav1.Time{Time: time.Now().Add(-35 * time.Second)}), // notReady, waitTime = t - 35s + generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), // notReady, no wait time because it does not have available condition yet, + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-2", cluster2), // Ready + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1909,9 +1946,11 @@ func TestPickBindingsToRoll(t *testing.T) { }, "test unscheduled bindings with different waitTimes and check the wait time is correct": { // want the min wait time of unscheduled bindings that are not ready - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2, metav1.Time{Time: now.Add(-1 * time.Minute)}), // NotReady, waitTime = t - 60s - generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3, metav1.Time{Time: now.Add(-35 * time.Second)}), // NotReady, waitTime = t - 35s + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2, metav1.Time{Time: time.Now().Add(-1 * time.Minute)}), // NotReady, waitTime = t - 60s + generateNotTrackableClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3, metav1.Time{Time: time.Now().Add(-35 * time.Second)}), // NotReady, waitTime = t - 35s + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -1933,8 +1972,10 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 25 * time.Second, // minWaitTime = (t - 35 seconds) - (t - 60 seconds) = 25 seconds }, "test only one bound but is deleting binding - rollout blocked": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + } }, crp: clusterResourcePlacementForTest("test", createPlacementPolicyForTest(placementv1beta1.PickAllPlacementType, 0), @@ -1944,11 +1985,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantNeedRoll: false, }, "test policy change with MaxSurge specified, evict resources on unscheduled cluster - rollout allowed for one scheduled binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), - generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), + generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -1984,11 +2027,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: time.Second, }, "test policy change with MaxUnavailable specified, evict resources on unscheduled cluster - rollout allowed for one unscheduled binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2024,9 +2069,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test resource snapshot change with MaxUnavailable specified, evict resources on ready bound binding - rollout allowed for one ready bound binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2056,9 +2103,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test resource snapshot change with MaxUnavailable specified, evict resource on canBeReady binding - rollout blocked": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2088,9 +2137,11 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: time.Second, }, "test resource snapshot change with MaxUnavailable specified, evict resources on failed to apply bound binding - rollout allowed for one failed to apply bound binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2120,11 +2171,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: time.Second, }, "test upscale, evict resources from ready bound binding - rollout allowed for two new scheduled bindings": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2161,13 +2214,15 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test upscale with policy change MaxSurge specified, evict resources from canBeReady bound binding - rollout allowed for three new scheduled bindings": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), - generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), + generateCanBeReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster4), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2213,11 +2268,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: time.Second, }, "test upscale with resource change MaxUnavailable specified, evict resources from ready bound binding - rollout allowed for old bound and new scheduled bindings": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-2", cluster3), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-2", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-2", cluster3), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-2", cluster4), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2257,11 +2314,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test downscale, evict resources from ready unscheduled binding - rollout allowed for one unscheduled binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2290,11 +2349,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test downscale, evict resources from ready bound binding - rollout allowed for two unscheduled bindings to be deleted": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2323,13 +2384,15 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test downscale with policy change, evict unscheduled ready binding - rollout allowed for one unscheduled binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2367,13 +2430,15 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test downscale with policy change, evict unscheduled failed to apply binding - rollout allowed for new scheduled bindings": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), - generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), - generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster1)), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster2), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), + generateFailedToApplyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster5), + generateClusterResourceBinding(placementv1beta1.BindingStateScheduled, "snapshot-1", cluster6), + } }, latestResourceSnapshotName: "snapshot-1", crp: clusterResourcePlacementForTest("test", @@ -2411,11 +2476,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: defaultUnavailablePeriod * time.Second, }, "test downscale with resource snapshot change, evict ready bound binding - rollout allowed for one unscheduled binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2455,11 +2522,13 @@ func TestPickBindingsToRoll(t *testing.T) { wantWaitTime: 0, }, "test downscale with resource snapshot change, evict ready unscheduled binding - rollout allowed for one unscheduled binding, one bound binding": { - allBindings: []*placementv1beta1.ClusterResourceBinding{ - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), - setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3)), - generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + allBindingsFunc: func() []*placementv1beta1.ClusterResourceBinding { + return []*placementv1beta1.ClusterResourceBinding{ + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster1), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateBound, "snapshot-1", cluster2), + setDeletionTimeStampForBinding(generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster3)), + generateReadyClusterResourceBinding(placementv1beta1.BindingStateUnscheduled, "snapshot-1", cluster4), + } }, latestResourceSnapshotName: "snapshot-2", crp: clusterResourcePlacementForTest("test", @@ -2519,7 +2588,8 @@ func TestPickBindingsToRoll(t *testing.T) { Name: tt.latestResourceSnapshotName, }, } - gotUpdatedBindings, gotStaleUnselectedBindings, gotUpToDateBoundBindings, gotNeedRoll, gotWaitTime, err := r.pickBindingsToRoll(context.Background(), controller.ConvertCRBArrayToBindingObjs(tt.allBindings), resourceSnapshot, tt.crp, tt.matchedCROs, tt.matchedROs) + allBindings := tt.allBindingsFunc() + gotUpdatedBindings, gotStaleUnselectedBindings, gotUpToDateBoundBindings, gotNeedRoll, gotWaitTime, err := r.pickBindingsToRoll(context.Background(), controller.ConvertCRBArrayToBindingObjs(tt.allBindingsFunc()), resourceSnapshot, tt.crp, tt.matchedCROs, tt.matchedROs) if (err != nil) != (tt.wantErr != nil) || err != nil && !errors.Is(err, tt.wantErr) { t.Fatalf("pickBindingsToRoll() error = %v, wantErr %v", err, tt.wantErr) } @@ -2530,30 +2600,30 @@ func TestPickBindingsToRoll(t *testing.T) { wantTobeUpdatedBindings := make([]toBeUpdatedBinding, len(tt.wantTobeUpdatedBindings)) for i, index := range tt.wantTobeUpdatedBindings { // Unscheduled bindings are only removed in a single rollout cycle. - bindingSpec := tt.allBindings[index].GetBindingSpec() + bindingSpec := allBindings[index].GetBindingSpec() if bindingSpec.State != placementv1beta1.BindingStateUnscheduled { - wantTobeUpdatedBindings[i].currentBinding = tt.allBindings[index] - wantTobeUpdatedBindings[i].desiredBinding = tt.allBindings[index].DeepCopy() + wantTobeUpdatedBindings[i].currentBinding = allBindings[index] + wantTobeUpdatedBindings[i].desiredBinding = allBindings[index].DeepCopy() wantTobeUpdatedBindings[i].desiredBinding.SetBindingSpec(tt.wantDesiredBindingsSpec[index]) } else { - wantTobeUpdatedBindings[i].currentBinding = tt.allBindings[index] + wantTobeUpdatedBindings[i].currentBinding = allBindings[index] } } wantStaleUnselectedBindings := make([]toBeUpdatedBinding, len(tt.wantStaleUnselectedBindings)) for i, index := range tt.wantStaleUnselectedBindings { // Unscheduled bindings are only removed in a single rollout cycle. - bindingSpec := tt.allBindings[index].GetBindingSpec() + bindingSpec := allBindings[index].GetBindingSpec() if bindingSpec.State != placementv1beta1.BindingStateUnscheduled { - wantStaleUnselectedBindings[i].currentBinding = tt.allBindings[index] - wantStaleUnselectedBindings[i].desiredBinding = tt.allBindings[index].DeepCopy() + wantStaleUnselectedBindings[i].currentBinding = allBindings[index] + wantStaleUnselectedBindings[i].desiredBinding = allBindings[index].DeepCopy() wantStaleUnselectedBindings[i].desiredBinding.SetBindingSpec(tt.wantDesiredBindingsSpec[index]) } else { - wantStaleUnselectedBindings[i].currentBinding = tt.allBindings[index] + wantStaleUnselectedBindings[i].currentBinding = allBindings[index] } } wantUpToDateBoundBindings := make([]toBeUpdatedBinding, len(tt.wantUpToDateBoundBindings)) for i, index := range tt.wantUpToDateBoundBindings { - wantUpToDateBoundBindings[i].currentBinding = tt.allBindings[index] + wantUpToDateBoundBindings[i].currentBinding = allBindings[index] } if diff := cmp.Diff(wantTobeUpdatedBindings, gotUpdatedBindings, cmpOptions...); diff != "" { diff --git a/pkg/controllers/updaterun/controller.go b/pkg/controllers/updaterun/controller.go index 1d1dbcee7..df1e9dcc6 100644 --- a/pkg/controllers/updaterun/controller.go +++ b/pkg/controllers/updaterun/controller.go @@ -79,7 +79,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim } runObjRef := klog.KObj(updateRun) - // Remove waitTime from the updateRun status for AfterStageTask for type Approval. + // Remove waitTime from the updateRun status for BeforeStageTask and AfterStageTask for type Approval. removeWaitTimeFromUpdateRunStatus(updateRun) // Handle the deletion of the updateRun. @@ -104,15 +104,23 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim // Emit the update run status metric based on status conditions in the updateRun. defer emitUpdateRunStatusMetric(updateRun) + state := updateRun.GetUpdateRunSpec().State + var updatingStageIndex int var toBeUpdatedBindings, toBeDeletedBindings []placementv1beta1.BindingObj updateRunStatus := updateRun.GetUpdateRunStatus() initCond := meta.FindStatusCondition(updateRunStatus.Conditions, string(placementv1beta1.StagedUpdateRunConditionInitialized)) - if !condition.IsConditionStatusTrue(initCond, updateRun.GetGeneration()) { - if condition.IsConditionStatusFalse(initCond, updateRun.GetGeneration()) { + // Check if initialized regardless of generation. + // The updateRun spec fields are immutable except for the state field. When the state changes, + // the update run generation increments, but we don't need to reinitialize since initialization is a one-time setup. + if !(initCond != nil && initCond.Status == metav1.ConditionTrue) { + // Check if initialization failed for the current generation. + if initCond != nil && initCond.Status == metav1.ConditionFalse { klog.V(2).InfoS("The updateRun has failed to initialize", "errorMsg", initCond.Message, "updateRun", runObjRef) return runtime.Result{}, nil } + + // Initialize the updateRun. var initErr error if toBeUpdatedBindings, toBeDeletedBindings, initErr = r.initialize(ctx, updateRun); initErr != nil { klog.ErrorS(initErr, "Failed to initialize the updateRun", "updateRun", runObjRef) @@ -122,10 +130,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim } return runtime.Result{}, initErr } - updatingStageIndex = 0 // start from the first stage. - klog.V(2).InfoS("Initialized the updateRun", "updateRun", runObjRef) + updatingStageIndex = 0 // start from the first stage (typically for Initialize or Run states). + klog.V(2).InfoS("Initialized the updateRun", "state", state, "updateRun", runObjRef) } else { - klog.V(2).InfoS("The updateRun is initialized", "updateRun", runObjRef) + klog.V(2).InfoS("The updateRun is initialized", "state", state, "updateRun", runObjRef) // Check if the updateRun is finished. finishedCond := meta.FindStatusCondition(updateRunStatus.Conditions, string(placementv1beta1.StagedUpdateRunConditionSucceeded)) if condition.IsConditionStatusTrue(finishedCond, updateRun.GetGeneration()) || condition.IsConditionStatusFalse(finishedCond, updateRun.GetGeneration()) { @@ -151,28 +159,32 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim } // Execute the updateRun. - klog.V(2).InfoS("Continue to execute the updateRun", "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) - finished, waitTime, execErr := r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings) - if errors.Is(execErr, errStagedUpdatedAborted) { - // errStagedUpdatedAborted cannot be retried. - return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, execErr.Error()) - } + if state == placementv1beta1.StateRun { + klog.V(2).InfoS("Continue to execute the updateRun", "state", state, "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) + finished, waitTime, execErr := r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings) + if errors.Is(execErr, errStagedUpdatedAborted) { + // errStagedUpdatedAborted cannot be retried. + return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, execErr.Error()) + } - if finished { - klog.V(2).InfoS("The updateRun is completed", "updateRun", runObjRef) - return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun) - } + if finished { + klog.V(2).InfoS("The updateRun is completed", "updateRun", runObjRef) + return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun) + } - // The execution is not finished yet or it encounters a retriable error. - // We need to record the status and requeue. - if updateErr := r.recordUpdateRunStatus(ctx, updateRun); updateErr != nil { - return runtime.Result{}, updateErr - } - klog.V(2).InfoS("The updateRun is not finished yet", "requeueWaitTime", waitTime, "execErr", execErr, "updateRun", runObjRef) - if execErr != nil { - return runtime.Result{}, execErr + // The execution is not finished yet or it encounters a retriable error. + // We need to record the status and requeue. + if updateErr := r.recordUpdateRunStatus(ctx, updateRun); updateErr != nil { + return runtime.Result{}, updateErr + } + klog.V(2).InfoS("The updateRun is not finished yet", "requeueWaitTime", waitTime, "execErr", execErr, "updateRun", runObjRef) + if execErr != nil { + return runtime.Result{}, execErr + } + return runtime.Result{Requeue: true, RequeueAfter: waitTime}, nil } - return runtime.Result{Requeue: true, RequeueAfter: waitTime}, nil + klog.V(2).InfoS("The updateRun is initialized but not executed, waiting to execute", "state", state, "updateRun", runObjRef) + return runtime.Result{}, nil } // handleDelete handles the deletion of the updateRun object. @@ -455,10 +467,15 @@ func emitUpdateRunStatusMetric(updateRun placementv1beta1.UpdateRunObj) { } func removeWaitTimeFromUpdateRunStatus(updateRun placementv1beta1.UpdateRunObj) { - // Remove waitTime from the updateRun status for AfterStageTask for type Approval. + // Remove waitTime from the updateRun status for BeforeStageTask and AfterStageTask for type Approval. updateRunStatus := updateRun.GetUpdateRunStatus() if updateRunStatus.UpdateStrategySnapshot != nil { for i := range updateRunStatus.UpdateStrategySnapshot.Stages { + for j := range updateRunStatus.UpdateStrategySnapshot.Stages[i].BeforeStageTasks { + if updateRunStatus.UpdateStrategySnapshot.Stages[i].BeforeStageTasks[j].Type == placementv1beta1.StageTaskTypeApproval { + updateRunStatus.UpdateStrategySnapshot.Stages[i].BeforeStageTasks[j].WaitTime = nil + } + } for j := range updateRunStatus.UpdateStrategySnapshot.Stages[i].AfterStageTasks { if updateRunStatus.UpdateStrategySnapshot.Stages[i].AfterStageTasks[j].Type == placementv1beta1.StageTaskTypeApproval { updateRunStatus.UpdateStrategySnapshot.Stages[i].AfterStageTasks[j].WaitTime = nil diff --git a/pkg/controllers/updaterun/controller_integration_test.go b/pkg/controllers/updaterun/controller_integration_test.go index b12859651..6453d5617 100644 --- a/pkg/controllers/updaterun/controller_integration_test.go +++ b/pkg/controllers/updaterun/controller_integration_test.go @@ -272,6 +272,16 @@ func generateMetricsLabels( } } +func generateInitializationSucceededMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionInitialized), + string(metav1.ConditionTrue), condition.UpdateRunInitializeSucceededReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + func generateInitializationFailedMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { return &prometheusclientmodel.Metric{ Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionInitialized), @@ -341,6 +351,7 @@ func generateTestClusterStagedUpdateRun() *placementv1beta1.ClusterStagedUpdateR PlacementName: testCRPName, ResourceSnapshotIndex: testResourceSnapshotIndex, StagedUpdateStrategyName: testUpdateStrategyName, + State: placementv1beta1.StateRun, }, } } @@ -505,6 +516,11 @@ func generateTestClusterStagedUpdateStrategy() *placementv1beta1.ClusterStagedUp }, }, SortingLabelKey: &sortingKey, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, AfterStageTasks: []placementv1beta1.StageTask{ { Type: placementv1beta1.StageTaskTypeTimedWait, @@ -526,6 +542,11 @@ func generateTestClusterStagedUpdateStrategy() *placementv1beta1.ClusterStagedUp }, }, // no sortingLabelKey, should sort by cluster name + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, AfterStageTasks: []placementv1beta1.StageTask{ { Type: placementv1beta1.StageTaskTypeApproval, @@ -543,7 +564,7 @@ func generateTestClusterStagedUpdateStrategy() *placementv1beta1.ClusterStagedUp } } -func generateTestClusterStagedUpdateStrategyWithSingleStage(afterStageTasks []placementv1beta1.StageTask) *placementv1beta1.ClusterStagedUpdateStrategy { +func generateTestClusterStagedUpdateStrategyWithSingleStage(beforeStageTasks, afterStageTasks []placementv1beta1.StageTask) *placementv1beta1.ClusterStagedUpdateStrategy { return &placementv1beta1.ClusterStagedUpdateStrategy{ ObjectMeta: metav1.ObjectMeta{ Name: testUpdateStrategyName, @@ -551,9 +572,10 @@ func generateTestClusterStagedUpdateStrategyWithSingleStage(afterStageTasks []pl Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { - Name: "stage1", - LabelSelector: &metav1.LabelSelector{}, // Select all clusters. - AfterStageTasks: afterStageTasks, + Name: "stage1", + LabelSelector: &metav1.LabelSelector{}, // Select all clusters. + BeforeStageTasks: beforeStageTasks, + AfterStageTasks: afterStageTasks, }, }, }, @@ -724,9 +746,9 @@ func generateTrueCondition(obj client.Object, condType any) metav1.Condition { case placementv1beta1.StageTaskConditionWaitTimeElapsed: reason = condition.AfterStageTaskWaitTimeElapsedReason case placementv1beta1.StageTaskConditionApprovalRequestCreated: - reason = condition.AfterStageTaskApprovalRequestCreatedReason + reason = condition.StageTaskApprovalRequestCreatedReason case placementv1beta1.StageTaskConditionApprovalRequestApproved: - reason = condition.AfterStageTaskApprovalRequestApprovedReason + reason = condition.StageTaskApprovalRequestApprovedReason } typeStr = string(cond) case placementv1beta1.ApprovalRequestConditionType: @@ -796,23 +818,8 @@ func generateFalseCondition(obj client.Object, condType any) metav1.Condition { } } -func generateFalseProgressingCondition(obj client.Object, condType any, succeeded bool) metav1.Condition { +func generateFalseProgressingCondition(obj client.Object, condType any, reason string) metav1.Condition { falseCond := generateFalseCondition(obj, condType) - reason := "" - switch condType { - case placementv1beta1.StagedUpdateRunConditionProgressing: - if succeeded { - reason = condition.UpdateRunSucceededReason - } else { - reason = condition.UpdateRunFailedReason - } - case placementv1beta1.StageUpdatingConditionProgressing: - if succeeded { - reason = condition.StageUpdatingSucceededReason - } else { - reason = condition.StageUpdatingFailedReason - } - } falseCond.Reason = reason return falseCond } diff --git a/pkg/controllers/updaterun/controller_test.go b/pkg/controllers/updaterun/controller_test.go index 1d671907b..fdc3fa5f7 100644 --- a/pkg/controllers/updaterun/controller_test.go +++ b/pkg/controllers/updaterun/controller_test.go @@ -912,7 +912,7 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }, }, }, - "should remove waitTime from Approval tasks only": { + "should remove waitTime from Approval tasks only for AfterStageTasks": { inputUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ Status: placementv1beta1.UpdateRunStatus{ UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ @@ -953,12 +953,51 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }, }, }, + "should remove waitTime from Approval tasks only for BeforeStageTasks": { + inputUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + WaitTime: &waitTime, + }, + }, + }, + }, + }, + }, + }, + wantUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + }, + }, + }, + }, + }, + }, "should handle multiple stages": { inputUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ Status: placementv1beta1.UpdateRunStatus{ UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + WaitTime: &waitTime, + }, + }, AfterStageTasks: []placementv1beta1.StageTask{ { Type: placementv1beta1.StageTaskTypeApproval, @@ -978,6 +1017,14 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }, }, }, + { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + WaitTime: &waitTime, + }, + }, + }, }, }, }, @@ -987,6 +1034,11 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, AfterStageTasks: []placementv1beta1.StageTask{ { Type: placementv1beta1.StageTaskTypeApproval, @@ -1004,6 +1056,13 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }, }, }, + { + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + }, }, }, }, diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index 47bfef806..72e265ec2 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -62,34 +62,78 @@ func (r *Reconciler) execute( updateRun placementv1beta1.UpdateRunObj, updatingStageIndex int, toBeUpdatedBindings, toBeDeletedBindings []placementv1beta1.BindingObj, -) (bool, time.Duration, error) { +) (finished bool, waitTime time.Duration, err error) { + updateRunStatus := updateRun.GetUpdateRunStatus() + var updatingStageStatus *placementv1beta1.StageUpdatingStatus + + // Set up defer function to handle errStagedUpdatedAborted. + defer func() { + if errors.Is(err, errStagedUpdatedAborted) { + if updatingStageStatus != nil { + markStageUpdatingFailed(updatingStageStatus, updateRun.GetGeneration(), err.Error()) + } else { + // Handle deletion stage case. + markStageUpdatingFailed(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration(), err.Error()) + } + } + }() + // Mark updateRun as progressing if it's not already marked as waiting or stuck. // This avoids triggering an unnecessary in-memory transition from stuck (waiting) -> progressing -> stuck (waiting), // which would update the lastTransitionTime even though the status hasn't effectively changed. markUpdateRunProgressingIfNotWaitingOrStuck(updateRun) - - updateRunStatus := updateRun.GetUpdateRunStatus() if updatingStageIndex < len(updateRunStatus.StagesStatus) { - maxConcurrency, err := calculateMaxConcurrencyValue(updateRunStatus, updatingStageIndex) + updatingStageStatus = &updateRunStatus.StagesStatus[updatingStageIndex] + approved, err := r.checkBeforeStageTasksStatus(ctx, updatingStageIndex, updateRun) if err != nil { return false, 0, err } - updatingStage := &updateRunStatus.StagesStatus[updatingStageIndex] - waitTime, execErr := r.executeUpdatingStage(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, maxConcurrency) - if errors.Is(execErr, errStagedUpdatedAborted) { - markStageUpdatingFailed(updatingStage, updateRun.GetGeneration(), execErr.Error()) - return true, waitTime, execErr + if !approved { + markStageUpdatingWaiting(updatingStageStatus, updateRun.GetGeneration(), "Not all before-stage tasks are completed, waiting for approval") + markUpdateRunWaiting(updateRun, fmt.Sprintf(condition.UpdateRunWaitingMessageFmt, "before-stage", updatingStageStatus.StageName)) + return false, stageUpdatingWaitTime, nil } + maxConcurrency, err := calculateMaxConcurrencyValue(updateRunStatus, updatingStageIndex) + if err != nil { + return false, 0, err + } + waitTime, err = r.executeUpdatingStage(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, maxConcurrency) // The execution has not finished yet. - return false, waitTime, execErr + return false, waitTime, err } // All the stages have finished, now start the delete stage. - finished, execErr := r.executeDeleteStage(ctx, updateRun, toBeDeletedBindings) - if errors.Is(execErr, errStagedUpdatedAborted) { - markStageUpdatingFailed(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration(), execErr.Error()) - return true, 0, execErr + finished, err = r.executeDeleteStage(ctx, updateRun, toBeDeletedBindings) + return finished, clusterUpdatingWaitTime, err +} + +// checkBeforeStageTasksStatus checks if the before stage tasks have finished. +// It returns if the before stage tasks have finished or error if the before stage tasks failed. +func (r *Reconciler) checkBeforeStageTasksStatus(ctx context.Context, updatingStageIndex int, updateRun placementv1beta1.UpdateRunObj) (bool, error) { + updateRunRef := klog.KObj(updateRun) + updateRunStatus := updateRun.GetUpdateRunStatus() + updatingStage := &updateRunStatus.UpdateStrategySnapshot.Stages[updatingStageIndex] + if updatingStage.BeforeStageTasks == nil { + klog.V(2).InfoS("There is no before stage task for this stage", "stage", updatingStage.Name, "updateRun", updateRunRef) + return true, nil } - return finished, clusterUpdatingWaitTime, execErr + + updatingStageStatus := &updateRunStatus.StagesStatus[updatingStageIndex] + for i, task := range updatingStage.BeforeStageTasks { + switch task.Type { + case placementv1beta1.StageTaskTypeApproval: + approved, err := r.handleStageApprovalTask(ctx, &updatingStageStatus.BeforeStageTaskStatus[i], updatingStage, updateRun, placementv1beta1.BeforeStageTaskLabelValue) + if err != nil { + return false, err + } + return approved, nil // Ideally there should be only one approval task in before stage tasks. + default: + // Approval is the only supported before stage task. + unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("found unsupported task type in before stage tasks: %s", task.Type)) + klog.ErrorS(unexpectedErr, "Task type is not supported in before stage tasks", "stage", updatingStage.Name, "updateRun", updateRunRef, "taskType", task.Type) + return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) + } + } + return true, nil } // executeUpdatingStage executes a single updating stage by updating the bindings. @@ -241,31 +285,45 @@ func (r *Reconciler) executeUpdatingStage( } if finishedClusterCount == len(updatingStageStatus.Clusters) { - // All the clusters in the stage have been updated. - markUpdateRunWaiting(updateRun, updatingStageStatus.StageName) - markStageUpdatingWaiting(updatingStageStatus, updateRun.GetGeneration()) - klog.V(2).InfoS("The stage has finished all cluster updating", "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) - // Check if the after stage tasks are ready. - approved, waitTime, err := r.checkAfterStageTasksStatus(ctx, updatingStageIndex, updateRun) - if err != nil { - return 0, err - } - if approved { - markUpdateRunProgressing(updateRun) - markStageUpdatingSucceeded(updatingStageStatus, updateRun.GetGeneration()) - // No need to wait to get to the next stage. - return 0, nil - } - // The after stage tasks are not ready yet. - if waitTime < 0 { - waitTime = stageUpdatingWaitTime - } - return waitTime, nil + return r.handleStageCompletion(ctx, updatingStageIndex, updateRun, updatingStageStatus) } + // Some clusters are still updating. return clusterUpdatingWaitTime, nil } +// handleStageCompletion handles the completion logic when all clusters in a stage are finished. +// Returns the wait time and any error encountered. +func (r *Reconciler) handleStageCompletion( + ctx context.Context, + updatingStageIndex int, + updateRun placementv1beta1.UpdateRunObj, + updatingStageStatus *placementv1beta1.StageUpdatingStatus, +) (time.Duration, error) { + updateRunRef := klog.KObj(updateRun) + + // All the clusters in the stage have been updated. + markUpdateRunWaiting(updateRun, fmt.Sprintf(condition.UpdateRunWaitingMessageFmt, "after-stage", updatingStageStatus.StageName)) + markStageUpdatingWaiting(updatingStageStatus, updateRun.GetGeneration(), "All clusters in the stage are updated, waiting for after-stage tasks to complete") + klog.V(2).InfoS("The stage has finished all cluster updating", "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) + // Check if the after stage tasks are ready. + approved, waitTime, err := r.checkAfterStageTasksStatus(ctx, updatingStageIndex, updateRun) + if err != nil { + return 0, err + } + if approved { + markUpdateRunProgressing(updateRun) + markStageUpdatingSucceeded(updatingStageStatus, updateRun.GetGeneration()) + // No need to wait to get to the next stage. + return 0, nil + } + // The after stage tasks are not ready yet. + if waitTime < 0 { + waitTime = stageUpdatingWaitTime + } + return waitTime, nil +} + // executeDeleteStage executes the delete stage by deleting the bindings. func (r *Reconciler) executeDeleteStage( ctx context.Context, @@ -360,59 +418,11 @@ func (r *Reconciler) checkAfterStageTasksStatus(ctx context.Context, updatingSta klog.V(2).InfoS("The after stage wait task has completed", "stage", updatingStage.Name, "updateRun", updateRunRef) } case placementv1beta1.StageTaskTypeApproval: - afterStageTaskApproved := condition.IsConditionStatusTrue(meta.FindStatusCondition(updatingStageStatus.AfterStageTaskStatus[i].Conditions, string(placementv1beta1.StageTaskConditionApprovalRequestApproved)), updateRun.GetGeneration()) - if afterStageTaskApproved { - // The afterStageTask has been approved. - continue + approved, err := r.handleStageApprovalTask(ctx, &updatingStageStatus.AfterStageTaskStatus[i], updatingStage, updateRun, placementv1beta1.AfterStageTaskLabelValue) + if err != nil { + return false, -1, err } - // Check if the approval request has been created. - approvalRequest := buildApprovalRequestObject(types.NamespacedName{Name: updatingStageStatus.AfterStageTaskStatus[i].ApprovalRequestName, Namespace: updateRun.GetNamespace()}, updatingStage.Name, updateRun.GetName()) - requestRef := klog.KObj(approvalRequest) - if err := r.Client.Create(ctx, approvalRequest); err != nil { - if apierrors.IsAlreadyExists(err) { - // The approval task already exists. - markAfterStageRequestCreated(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.GetGeneration()) - if err = r.Client.Get(ctx, client.ObjectKeyFromObject(approvalRequest), approvalRequest); err != nil { - klog.ErrorS(err, "Failed to get the already existing approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - return false, -1, controller.NewAPIServerError(true, err) - } - approvalRequestSpec := approvalRequest.GetApprovalRequestSpec() - if approvalRequestSpec.TargetStage != updatingStage.Name || approvalRequestSpec.TargetUpdateRun != updateRun.GetName() { - unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the approval request task `%s/%s` is targeting update run `%s/%s` and stage `%s`", approvalRequest.GetNamespace(), approvalRequest.GetName(), approvalRequest.GetNamespace(), approvalRequestSpec.TargetUpdateRun, approvalRequestSpec.TargetStage)) - klog.ErrorS(unexpectedErr, "Found an approval request targeting wrong stage", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - return false, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) - } - approvalRequestStatus := approvalRequest.GetApprovalRequestStatus() - approvalAccepted := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequestStatus.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.GetGeneration()) - approved := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequestStatus.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)), approvalRequest.GetGeneration()) - if !approvalAccepted && !approved { - klog.V(2).InfoS("The approval request has not been approved yet", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - passed = false - continue - } - if approved { - klog.V(2).InfoS("The approval request has been approved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - if !approvalAccepted { - if err = r.updateApprovalRequestAccepted(ctx, approvalRequest); err != nil { - klog.ErrorS(err, "Failed to accept the approved approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - // retriable err - return false, -1, err - } - } - } else { - // Approved state should not change once the approval is accepted. - klog.V(2).InfoS("The approval request has been approval-accepted, ignoring changing back to unapproved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - } - markAfterStageRequestApproved(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.GetGeneration()) - } else { - // retriable error - klog.ErrorS(err, "Failed to create the approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - return false, -1, controller.NewAPIServerError(false, err) - } - } else { - // The approval request has been created for the first time. - klog.V(2).InfoS("The approval request has been created", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) - markAfterStageRequestCreated(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.GetGeneration()) + if !approved { passed = false } } @@ -423,6 +433,75 @@ func (r *Reconciler) checkAfterStageTasksStatus(ctx context.Context, updatingSta return passed, afterStageWaitTime, nil } +// handleStageApprovalTask handles the approval task logic for before or after stage tasks. +// It returns true if the task is approved, false otherwise, and any error encountered. +func (r *Reconciler) handleStageApprovalTask( + ctx context.Context, + stageTaskStatus *placementv1beta1.StageTaskStatus, + updatingStage *placementv1beta1.StageConfig, + updateRun placementv1beta1.UpdateRunObj, + stageTaskType string, +) (bool, error) { + updateRunRef := klog.KObj(updateRun) + + stageTaskApproved := condition.IsConditionStatusTrue(meta.FindStatusCondition(stageTaskStatus.Conditions, string(placementv1beta1.StageTaskConditionApprovalRequestApproved)), updateRun.GetGeneration()) + if stageTaskApproved { + // The stageTask has been approved. + return true, nil + } + + // Check if the approval request has been created. + approvalRequest := buildApprovalRequestObject(types.NamespacedName{Name: stageTaskStatus.ApprovalRequestName, Namespace: updateRun.GetNamespace()}, updatingStage.Name, updateRun.GetName(), stageTaskType) + requestRef := klog.KObj(approvalRequest) + if err := r.Client.Create(ctx, approvalRequest); err != nil { + if apierrors.IsAlreadyExists(err) { + // The approval task already exists. + markStageTaskRequestCreated(stageTaskStatus, updateRun.GetGeneration()) + if err = r.Client.Get(ctx, client.ObjectKeyFromObject(approvalRequest), approvalRequest); err != nil { + klog.ErrorS(err, "Failed to get the already existing approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + return false, controller.NewAPIServerError(true, err) + } + approvalRequestSpec := approvalRequest.GetApprovalRequestSpec() + if approvalRequestSpec.TargetStage != updatingStage.Name || approvalRequestSpec.TargetUpdateRun != updateRun.GetName() { + unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the approval request task `%s/%s` is targeting update run `%s/%s` and stage `%s`, want target update run `%s/%s and stage `%s`", approvalRequest.GetNamespace(), approvalRequest.GetName(), approvalRequest.GetNamespace(), approvalRequestSpec.TargetUpdateRun, approvalRequestSpec.TargetStage, approvalRequest.GetNamespace(), updateRun.GetName(), updatingStage.Name)) + klog.ErrorS(unexpectedErr, "Found an approval request targeting wrong stage", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) + } + approvalRequestStatus := approvalRequest.GetApprovalRequestStatus() + approvalAccepted := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequestStatus.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.GetGeneration()) + approved := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequestStatus.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)), approvalRequest.GetGeneration()) + if !approvalAccepted && !approved { + klog.V(2).InfoS("The approval request has not been approved yet", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + return false, nil + } + if approved { + klog.V(2).InfoS("The approval request has been approved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + if !approvalAccepted { + if err = r.updateApprovalRequestAccepted(ctx, approvalRequest); err != nil { + klog.ErrorS(err, "Failed to accept the approved approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + // retriable err + return false, err + } + } + } else { + // Approved state should not change once the approval is accepted. + klog.V(2).InfoS("The approval request has been approval-accepted, ignoring changing back to unapproved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + } + markStageTaskRequestApproved(stageTaskStatus, updateRun.GetGeneration()) + } else { + // retriable error + klog.ErrorS(err, "Failed to create the approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + return false, controller.NewAPIServerError(false, err) + } + } else { + // The approval request has been created for the first time. + klog.V(2).InfoS("The approval request has been created", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + markStageTaskRequestCreated(stageTaskStatus, updateRun.GetGeneration()) + return false, nil + } + return true, nil +} + // updateBindingRolloutStarted updates the binding status to indicate the rollout has started. func (r *Reconciler) updateBindingRolloutStarted(ctx context.Context, binding placementv1beta1.BindingObj, updateRun placementv1beta1.UpdateRunObj) error { // first reset the condition to reflect the latest lastTransitionTime @@ -540,9 +619,9 @@ func checkClusterUpdateResult( return false, nil } -// buildApprovalRequestObject creates an approval request object for after-stage tasks. +// buildApprovalRequestObject creates an approval request object for before-stage or after-stage tasks. // It returns a ClusterApprovalRequest if namespace is empty, otherwise returns an ApprovalRequest. -func buildApprovalRequestObject(namespacedName types.NamespacedName, stageName, updateRunName string) placementv1beta1.ApprovalRequestObj { +func buildApprovalRequestObject(namespacedName types.NamespacedName, stageName, updateRunName, stageTaskType string) placementv1beta1.ApprovalRequestObj { var approvalRequest placementv1beta1.ApprovalRequestObj if namespacedName.Namespace == "" { approvalRequest = &placementv1beta1.ClusterApprovalRequest{ @@ -551,6 +630,7 @@ func buildApprovalRequestObject(namespacedName types.NamespacedName, stageName, Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: stageTaskType, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -567,6 +647,7 @@ func buildApprovalRequestObject(namespacedName types.NamespacedName, stageName, Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: stageTaskType, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -616,14 +697,14 @@ func markUpdateRunStuck(updateRun placementv1beta1.UpdateRunObj, stageName, clus } // markUpdateRunWaiting marks the updateRun as waiting in memory. -func markUpdateRunWaiting(updateRun placementv1beta1.UpdateRunObj, stageName string) { +func markUpdateRunWaiting(updateRun placementv1beta1.UpdateRunObj, message string) { updateRunStatus := updateRun.GetUpdateRunStatus() meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), Status: metav1.ConditionFalse, ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunWaitingReason, - Message: fmt.Sprintf("The updateRun is waiting for after-stage tasks in stage %s to complete", stageName), + Message: message, }) } @@ -642,13 +723,13 @@ func markStageUpdatingStarted(stageUpdatingStatus *placementv1beta1.StageUpdatin } // markStageUpdatingWaiting marks the stage updating status as waiting in memory. -func markStageUpdatingWaiting(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64) { +func markStageUpdatingWaiting(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64, message string) { meta.SetStatusCondition(&stageUpdatingStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StageUpdatingConditionProgressing), Status: metav1.ConditionFalse, ObservedGeneration: generation, Reason: condition.StageUpdatingWaitingReason, - Message: "All clusters in the stage are updated, waiting for after-stage tasks to complete", + Message: message, }) } @@ -727,24 +808,24 @@ func markClusterUpdatingFailed(clusterUpdatingStatus *placementv1beta1.ClusterUp }) } -// markAfterStageRequestCreated marks the Approval after stage task as ApprovalRequestCreated in memory. -func markAfterStageRequestCreated(afterStageTaskStatus *placementv1beta1.StageTaskStatus, generation int64) { - meta.SetStatusCondition(&afterStageTaskStatus.Conditions, metav1.Condition{ +// markStageTaskRequestCreated marks the Approval for the before or after stage task as ApprovalRequestCreated in memory. +func markStageTaskRequestCreated(stageTaskStatus *placementv1beta1.StageTaskStatus, generation int64) { + meta.SetStatusCondition(&stageTaskStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StageTaskConditionApprovalRequestCreated), Status: metav1.ConditionTrue, ObservedGeneration: generation, - Reason: condition.AfterStageTaskApprovalRequestCreatedReason, + Reason: condition.StageTaskApprovalRequestCreatedReason, Message: "ApprovalRequest object is created", }) } -// markAfterStageRequestApproved marks the Approval after stage task as Approved in memory. -func markAfterStageRequestApproved(afterStageTaskStatus *placementv1beta1.StageTaskStatus, generation int64) { - meta.SetStatusCondition(&afterStageTaskStatus.Conditions, metav1.Condition{ +// markStageTaskRequestApproved marks the Approval for the before or after stage task as Approved in memory. +func markStageTaskRequestApproved(stageTaskStatus *placementv1beta1.StageTaskStatus, generation int64) { + meta.SetStatusCondition(&stageTaskStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StageTaskConditionApprovalRequestApproved), Status: metav1.ConditionTrue, ObservedGeneration: generation, - Reason: condition.AfterStageTaskApprovalRequestApprovedReason, + Reason: condition.StageTaskApprovalRequestApprovedReason, Message: "ApprovalRequest object is approved", }) } diff --git a/pkg/controllers/updaterun/execution_integration_test.go b/pkg/controllers/updaterun/execution_integration_test.go index 2fc3c19ea..481c8b58c 100644 --- a/pkg/controllers/updaterun/execution_integration_test.go +++ b/pkg/controllers/updaterun/execution_integration_test.go @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + promclient "github.com/prometheus/client_model/go" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -155,17 +156,65 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { }) Context("Cluster staged update run should update clusters one by one", Ordered, func() { + var wantApprovalRequest *placementv1beta1.ClusterApprovalRequest BeforeAll(func() { By("Creating a new clusterStagedUpdateRun") Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) - By("Validating the initialization succeeded and the execution started") + By("Validating the initialization succeeded and the execution has not started") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - wantStatus = generateExecutionStartedStatus(updateRun, initialized) + wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + By("Validating the first beforeStage approvalRequest has been created") + wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[0].BeforeStageTaskStatus[0].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[0].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) + }) + + It("Should not start rolling out 1st stage", func() { + By("Validating the 1st clusterResourceBinding is not updated to Bound") + binding := resourceBindings[numTargetClusters-1] // cluster-9 + validateNotBoundBindingState(ctx, binding) + + By("Validating the 1st stage does not have startTime set") + Expect(updateRun.Status.StagesStatus[0].StartTime).Should(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) + }) + + It("Should accept the approval request and start to rollout 1st stage", func() { + By("Approving the approvalRequest") + approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) + + By("Validating the approvalRequest has ApprovalAccepted status") + Eventually(func() (bool, error) { + var approvalRequest placementv1beta1.ClusterApprovalRequest + if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { + return false, err + } + return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil + }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + // Approval task has been approved. + wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) }) It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -177,6 +226,9 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + // 1st stage started. + wantStatus = generateExecutionStartedStatus(updateRun, wantStatus) + By("Validating the 1st cluster has succeeded and 2nd cluster has started") wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) @@ -186,7 +238,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 2nd cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -204,7 +256,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 3rd cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -222,7 +274,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 4th cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -240,7 +292,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 5th cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -252,12 +304,13 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - By("Validating the 5th cluster has succeeded and stage waiting for AfterStageTasks") + By("Validating the 5th cluster has succeeded and 1st stage has completed and is waiting for AfterStageTasks") + // 5th cluster succeeded. wantStatus.StagesStatus[0].Clusters[4].Conditions = append(wantStatus.StagesStatus[0].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[0].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. - wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, - generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + // Now waiting for after stage tasks of 1st stage. + meta.SetStatusCondition(&wantStatus.StagesStatus[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") @@ -272,6 +325,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -293,13 +347,13 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) // 1st stage completed, mark progressing condition reason as succeeded and add succeeded condition. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) - // 2nd stage started. - wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) - // 1st cluster in 2nd stage started. - wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - wantStatus.Conditions[1] = generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + // 2nd stage waiting for before stage tasks. + wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Validating the 1st stage has endTime set") @@ -316,10 +370,62 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Expect(approvalCreateTime.Before(waitEndTime)).Should(BeTrue()) By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) + }) + + It("Should create approval request before 2nd stage", func() { + By("Validating the approvalRequest has been created") + wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[1].BeforeStageTaskStatus[0].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[1].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[1].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) + }) + + It("Should not start rolling out 2nd stage", func() { + By("Validating the 1st clusterResourceBinding is not updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBoundBindingState(ctx, binding) + + By("Validating the 1st stage does not have startTime set") + Expect(updateRun.Status.StagesStatus[1].StartTime).Should(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) }) - It("Should mark the 1st cluster in the 2nd stage as succeeded after marking the binding available", func() { + It("Should accept the approval request and start to rollout 2nd stage", func() { + By("Approving the approvalRequest") + approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) + + By("Validating the approvalRequest has ApprovalAccepted status") + Eventually(func() (bool, error) { + var approvalRequest placementv1beta1.ClusterApprovalRequest + if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { + return false, err + } + return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil + }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + // Approval task has been approved. + wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) + }) + + It("Should mark the 1st cluster in the 2nd stage as succeeded after approving request and marking the binding available", func() { By("Validating the 1st clusterResourceBinding is updated to Bound") binding := resourceBindings[0] // cluster-0 validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) @@ -327,6 +433,11 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { By("Updating the 1st clusterResourceBinding to Available") meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + // 2nd stage started. + wantStatus.StagesStatus[1].Conditions[0] = generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + // 1st cluster started. + wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) By("Validating the 1st cluster has succeeded and 2nd cluster has started") wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) @@ -408,7 +519,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { wantStatus.StagesStatus[1].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") @@ -423,6 +534,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[1].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -441,9 +553,9 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) wantStatus.StagesStatus[1].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) - wantStatus.StagesStatus[1].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[1].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) - wantStatus.Conditions[1] = generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) for i := range wantStatus.DeletionStageStatus.Clusters { @@ -487,7 +599,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { return fmt.Errorf("binding %s is not deleted", binding.Name) } if !apierrors.IsNotFound(err) { - return fmt.Errorf("Get binding %s does not return a not-found error: %w", binding.Name, err) + return fmt.Errorf("get binding %s does not return a not-found error: %w", binding.Name, err) } } return nil @@ -498,10 +610,10 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { wantStatus.DeletionStageStatus.Clusters[i].Conditions = append(wantStatus.DeletionStageStatus.Clusters[i].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) } // Mark the stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.DeletionStageStatus.Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -520,13 +632,13 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { By("Creating a new clusterStagedUpdateRun") Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) - By("Validating the initialization succeeded and the execution started") + By("Validating the initialization succeeded and the execution has not started") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - wantStatus = generateExecutionStartedStatus(updateRun, initialized) + wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) }) AfterAll(func() { @@ -535,6 +647,24 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { }) It("Should keep waiting for the 1st cluster while it's not available", func() { + By("Approving the approvalRequest") + approvalName := updateRun.Status.StagesStatus[0].BeforeStageTaskStatus[0].ApprovalRequestName + approveClusterApprovalRequest(ctx, approvalName) + + By("Validating the approvalRequest has ApprovalAccepted status") + Eventually(func() (bool, error) { + var approvalRequest placementv1beta1.ClusterApprovalRequest + if err := k8sClient.Get(ctx, types.NamespacedName{Name: approvalName}, &approvalRequest); err != nil { + return false, err + } + return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil + }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + // Approval task has been approved. + wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) + // 1st stage started. + wantStatus = generateExecutionStartedStatus(updateRun, wantStatus) + By("Validating the 1st clusterResourceBinding is updated to Bound") binding := resourceBindings[numTargetClusters-1] // cluster-9 validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) @@ -544,7 +674,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") By("Validating the updateRun is stuck in the 1st cluster of the 1st stage") - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) wantStatus.Conditions[1].Reason = condition.UpdateRunStuckReason validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, wantStatus, "") @@ -552,7 +682,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { It("Should emit stuck status metrics after time waiting for the 1st cluster reaches threshold", func() { By("Checking update run stuck metrics is emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateStuckMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun), generateStuckMetric(updateRun)) }) It("Should abort the execution if the binding has unexpected state", func() { @@ -566,14 +696,14 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { By("Validating the updateRun has failed") wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, false) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingFailedReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, false) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunFailedReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateStuckMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun), generateStuckMetric(updateRun), generateFailedMetric(updateRun)) }) }) }) @@ -602,7 +732,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { policySnapshot = generateTestClusterSchedulingPolicySnapshot(1, len(targetClusters)) resourceSnapshot = generateTestClusterResourceSnapshot() resourceSnapshot = generateTestClusterResourceSnapshot() - updateStrategy = generateTestClusterStagedUpdateStrategyWithSingleStage(nil) + updateStrategy = generateTestClusterStagedUpdateStrategyWithSingleStage(nil, nil) // Set smaller wait time for testing stageUpdatingWaitTime = time.Second * 3 @@ -739,13 +869,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Validating the 3rd cluster has succeeded and stage waiting for AfterStageTasks") wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -833,7 +963,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Validating the 3rd cluster has succeeded and stage waiting for AfterStageTasks") wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") @@ -845,13 +975,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -944,7 +1074,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) wantStatus.StagesStatus[0].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") @@ -959,6 +1089,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -977,13 +1108,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1073,13 +1204,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Validating the 3rd cluster has succeeded and stage waiting for AfterStageTasks") wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1129,7 +1260,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") By("Validating the updateRun is stuck in the 1st cluster of the 1st stage") - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) wantStatus.Conditions[1].Reason = condition.UpdateRunStuckReason validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, wantStatus, "") @@ -1142,8 +1273,14 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { }) Context("Cluster staged update run should recreate deleted approvalRequest", Ordered, func() { + var wantApprovalRequest *placementv1beta1.ClusterApprovalRequest BeforeAll(func() { By("Creating a strategy with single stage and both after stage tasks") + updateStrategy.Spec.Stages[0].BeforeStageTasks = []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + } updateStrategy.Spec.Stages[0].AfterStageTasks = []placementv1beta1.StageTask{ { Type: placementv1beta1.StageTaskTypeApproval, @@ -1162,10 +1299,93 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Creating a new clusterStagedUpdateRun") Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) - By("Validating the initialization succeeded and the execution started") + By("Validating the initialization succeeded and the execution has not started") initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) - wantStatus = generateExecutionStartedStatus(updateRun, initialized) + wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the approvalRequest has been created") + wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[0].BeforeStageTaskStatus[0].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[0].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + + }) + + It("Should not start rolling out", func() { + By("Validating the 1st clusterResourceBinding is not updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBoundBindingState(ctx, binding) + + By("Validating the 1st stage does not have startTime set") + Expect(updateRun.Status.StagesStatus[0].StartTime).Should(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) + }) + + It("Should start the 1st stage after approval request is approved", func() { + By("Validating the approvalRequest has been created") + approvalRequest := &placementv1beta1.ClusterApprovalRequest{} + wantApprovalRequest := &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[0].BeforeStageTaskStatus[0].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[0].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + + By("Deleting the approvalRequest") + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, approvalRequest)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, approvalRequest)).Should(Succeed()) + + By("Validating the approvalRequest has been recreated immediately") + validateApprovalRequestCreated(wantApprovalRequest) + + By("Approving the approvalRequest") + approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) + + By("Check the updateRun status") + wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) + wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + wantStatus.StagesStatus[0].Conditions[0] = generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with progressing reason. + meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Deleting the approvalRequest") + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, approvalRequest)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, approvalRequest)).Should(Succeed(), "failed to delete the approvalRequest") + + By("Validating the approvalRequest has not been recreated") + Eventually(func() bool { + return apierrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, approvalRequest)) + }, timeout, interval).Should(BeTrue(), "failed to ensure the approvalRequest is not recreated") + Consistently(func() bool { + return apierrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, approvalRequest)) + }, timeout, interval).Should(BeTrue(), "failed to ensure the approvalRequest is not recreated") }) It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -1215,7 +1435,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.StagesStatus[0].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) - wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) @@ -1228,6 +1448,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -1271,13 +1492,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Validating the 1st stage has completed") wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) // Need to have a longer wait time for the test to pass, because of the long wait time specified in the update strategy. timeout = time.Second * 90 @@ -1304,6 +1525,122 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { }, timeout, interval).Should(BeTrue(), "failed to ensure the approvalRequest is not recreated") }) }) + + Context("Cluster staged update run should update clusters one by one - different states (Initialize -> Run)", Ordered, func() { + var wantMetrics []*promclient.Metric + BeforeAll(func() { + By("Creating a new clusterStagedUpdateRun") + updateRun.Spec.State = placementv1beta1.StateInitialize + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the initialization succeeded and but not execution started") + wantStatus = generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateInitializationSucceededMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not start execution when the state is Initialize", func() { + By("Validating no execution has started") + Consistently(func() bool { + var currentUpdateRun placementv1beta1.ClusterStagedUpdateRun + if err := k8sClient.Get(ctx, types.NamespacedName{Name: updateRun.Name}, ¤tUpdateRun); err != nil { + return false + } + return meta.FindStatusCondition(currentUpdateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) == nil && + meta.FindStatusCondition(currentUpdateRun.Status.StagesStatus[0].Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)) == nil + }, timeout, interval).Should(BeTrue(), "execution has started unexpectedly") + + By("Validating the 1st clusterResourceBinding is updated to NOT Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBoundBindingState(ctx, binding) + }) + + It("Should start execution after changing the state to Run", func() { + By("Updating the updateRun state to Run") + updateRun.Spec.State = placementv1beta1.StateRun + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has started") + wantStatus = generateExecutionStartedStatus(updateRun, wantStatus) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 1st clusterResourceBinding is updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 1st clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 1st cluster has succeeded and 2nd cluster has started") + wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the 1st stage has startTime set") + Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 2nd cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 2nd clusterResourceBinding is updated to Bound") + binding := resourceBindings[1] // cluster-1 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 2nd clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 2nd cluster has succeeded and 3rd cluster has started") + wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 3rd cluster in the 1st stage as succeeded after marking the binding available and complete the updateRun", func() { + By("Validating the 3rd clusterResourceBinding is updated to Bound") + binding := resourceBindings[2] // cluster-2 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 3rd clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 3rd cluster has succeeded and stage waiting for AfterStageTasks") + wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + // 1st stage completed. + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) + wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) + // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) + // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. + wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason) + wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the 1st stage has endTime set") + Expect(updateRun.Status.StagesStatus[0].EndTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateSucceededMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + }) }) func validateBindingState(ctx context.Context, binding *placementv1beta1.ClusterResourceBinding, resourceSnapshotName string, updateRun *placementv1beta1.ClusterStagedUpdateRun, stage int) { @@ -1336,6 +1673,23 @@ func validateBindingState(ctx context.Context, binding *placementv1beta1.Cluster }, timeout, interval).Should(Succeed(), "failed to validate the binding state") } +func validateNotBoundBindingState(ctx context.Context, binding *placementv1beta1.ClusterResourceBinding) { + Consistently(func() error { + if err := k8sClient.Get(ctx, types.NamespacedName{Name: binding.Name}, binding); err != nil { + return err + } + + if binding.Spec.State == placementv1beta1.BindingStateBound { + return fmt.Errorf("binding %s is in Bound state, got %s", binding.Name, binding.Spec.State) + } + rolloutStartedCond := binding.GetCondition(string(placementv1beta1.ResourceBindingRolloutStarted)) + if condition.IsConditionStatusTrue(rolloutStartedCond, binding.Generation) { + return fmt.Errorf("binding %s rollout has started", binding.Name) + } + return nil + }, duration, interval).Should(Succeed(), "failed to validate the binding state") +} + func approveClusterApprovalRequest(ctx context.Context, approvalRequestName string) { Eventually(func() error { var approvalRequest placementv1beta1.ClusterApprovalRequest diff --git a/pkg/controllers/updaterun/execution_test.go b/pkg/controllers/updaterun/execution_test.go index ef26b283a..f43b8a803 100644 --- a/pkg/controllers/updaterun/execution_test.go +++ b/pkg/controllers/updaterun/execution_test.go @@ -19,6 +19,7 @@ package updaterun import ( "context" "errors" + "fmt" "strings" "testing" "time" @@ -345,22 +346,25 @@ func TestBuildApprovalRequestObject(t *testing.T) { namespacedName types.NamespacedName stageName string updateRunName string + stageTaskType string want placementv1beta1.ApprovalRequestObj }{ { name: "should create ClusterApprovalRequest when namespace is empty", namespacedName: types.NamespacedName{ - Name: "test-approval-request", + Name: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, "test-update-run", "test-stage"), Namespace: "", }, stageName: "test-stage", updateRunName: "test-update-run", + stageTaskType: placementv1beta1.BeforeStageTaskLabelValue, want: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-approval-request", + Name: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, "test-update-run", "test-stage"), Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: "test-stage", placementv1beta1.TargetUpdateRunLabel: "test-update-run", + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -373,18 +377,20 @@ func TestBuildApprovalRequestObject(t *testing.T) { { name: "should create namespaced ApprovalRequest when namespace is provided", namespacedName: types.NamespacedName{ - Name: "test-approval-request", + Name: fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, "test-update-run", "test-stage"), Namespace: "test-namespace", }, stageName: "test-stage", updateRunName: "test-update-run", + stageTaskType: placementv1beta1.AfterStageTaskLabelValue, want: &placementv1beta1.ApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-approval-request", + Name: fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, "test-update-run", "test-stage"), Namespace: "test-namespace", Labels: map[string]string{ placementv1beta1.TargetUpdatingStageNameLabel: "test-stage", placementv1beta1.TargetUpdateRunLabel: "test-update-run", + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", }, }, @@ -398,7 +404,7 @@ func TestBuildApprovalRequestObject(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - got := buildApprovalRequestObject(test.namespacedName, test.stageName, test.updateRunName) + got := buildApprovalRequestObject(test.namespacedName, test.stageName, test.updateRunName, test.stageTaskType) // Compare the whole objects using cmp.Diff with ignore options if diff := cmp.Diff(test.want, got); diff != "" { @@ -943,3 +949,259 @@ func TestCalculateMaxConcurrencyValue(t *testing.T) { }) } } + +func TestCheckBeforeStageTasksStatus_NegativeCases(t *testing.T) { + stageName := "stage-0" + testUpdateRunName = "test-update-run" + approvalRequestName := fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName) + tests := []struct { + name string + stageIndex int + updateRun *placementv1beta1.ClusterStagedUpdateRun + approvalRequest *placementv1beta1.ClusterApprovalRequest + wantErrMsg string + wantErrAborted bool + }{ + // Negative test cases only + { + name: "should return err if before stage task is TimedWait", + stageIndex: 0, + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: stageName, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeTimedWait, + }, + }, + }, + }, + }, + StagesStatus: []placementv1beta1.StageUpdatingStatus{ + { + StageName: stageName, + BeforeStageTaskStatus: []placementv1beta1.StageTaskStatus{ + { + Type: placementv1beta1.StageTaskTypeTimedWait, + }, + }, + }, + }, + }, + }, + wantErrMsg: fmt.Sprintf("found unsupported task type in before stage tasks: %s", placementv1beta1.StageTaskTypeTimedWait), + wantErrAborted: true, + }, + { + name: "should return err if Approval request has wrong target stage in spec", + stageIndex: 0, + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: testUpdateRunName, + }, + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: stageName, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + }, + }, + }, + StagesStatus: []placementv1beta1.StageUpdatingStatus{ + { + StageName: stageName, + BeforeStageTaskStatus: []placementv1beta1.StageTaskStatus{ + { + Type: placementv1beta1.StageTaskTypeApproval, + ApprovalRequestName: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName), + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.StageTaskConditionApprovalRequestCreated), + Status: metav1.ConditionTrue, + }, + }, + }, + }, + }, + }, + }, + }, + approvalRequest: &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: approvalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: testUpdateRunName, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: testUpdateRunName, + TargetStage: "stage-1", + }, + }, + wantErrMsg: fmt.Sprintf("the approval request task `/%s` is targeting update run `/%s` and stage `stage-1`", approvalRequestName, testUpdateRunName), + wantErrAborted: true, + }, + { + name: "should return err if Approval request has wrong target update run in spec", + stageIndex: 0, + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: testUpdateRunName, + }, + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: stageName, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + }, + }, + }, + StagesStatus: []placementv1beta1.StageUpdatingStatus{ + { + StageName: stageName, + BeforeStageTaskStatus: []placementv1beta1.StageTaskStatus{ + { + Type: placementv1beta1.StageTaskTypeApproval, + ApprovalRequestName: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName), + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.StageTaskConditionApprovalRequestCreated), + Status: metav1.ConditionTrue, + }, + }, + }, + }, + }, + }, + }, + }, + approvalRequest: &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName), + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: testUpdateRunName, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "wrong-update-run", + TargetStage: stageName, + }, + }, + wantErrMsg: fmt.Sprintf("the approval request task `/%s` is targeting update run `/wrong-update-run` and stage `%s`", approvalRequestName, stageName), + wantErrAborted: true, + }, + { + name: "should return err if cannot update Approval request that is approved as accepted", + stageIndex: 0, + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: testUpdateRunName, + }, + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: stageName, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + }, + }, + }, + StagesStatus: []placementv1beta1.StageUpdatingStatus{ + { + StageName: stageName, + BeforeStageTaskStatus: []placementv1beta1.StageTaskStatus{ + { + Type: placementv1beta1.StageTaskTypeApproval, + ApprovalRequestName: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName), + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.StageTaskConditionApprovalRequestCreated), + Status: metav1.ConditionTrue, + }, + }, + }, + }, + }, + }, + }, + }, + approvalRequest: &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, testUpdateRunName, stageName), + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: testUpdateRunName, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: testUpdateRunName, + TargetStage: stageName, + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + Status: metav1.ConditionTrue, + }, + }, + }, + }, + wantErrMsg: fmt.Sprintf("error returned by the API server: clusterapprovalrequests.placement.kubernetes-fleet.io \"%s\" not found", approvalRequestName), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objects := []client.Object{tt.updateRun} + if tt.approvalRequest != nil { + objects = append(objects, tt.approvalRequest) + } + objectsWithStatus := []client.Object{tt.updateRun} + scheme := runtime.NewScheme() + _ = placementv1beta1.AddToScheme(scheme) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource(objectsWithStatus...). + Build() + r := Reconciler{ + Client: fakeClient, + } + ctx := context.Background() + _, gotErr := r.checkBeforeStageTasksStatus(ctx, tt.stageIndex, tt.updateRun) + if gotErr == nil { + t.Fatalf("checkBeforeStageTasksStatus() want error but got nil") + } + if !strings.Contains(gotErr.Error(), tt.wantErrMsg) { + t.Fatalf("checkBeforeStageTasksStatus() error = %v, wantErr %v", gotErr, tt.wantErrMsg) + } + if tt.wantErrAborted && !errors.Is(gotErr, errStagedUpdatedAborted) { + t.Fatalf("checkBeforeStageTasksStatus() want aborted error but got different error: %v", gotErr) + } + }) + } +} diff --git a/pkg/controllers/updaterun/initialization.go b/pkg/controllers/updaterun/initialization.go index 574645003..223029c3a 100644 --- a/pkg/controllers/updaterun/initialization.go +++ b/pkg/controllers/updaterun/initialization.go @@ -292,7 +292,7 @@ func (r *Reconciler) generateStagesByStrategy( updateStrategySpec := updateStrategy.GetUpdateStrategySpec() updateRunStatus.UpdateStrategySnapshot = updateStrategySpec - // Remove waitTime from the updateRun status for AfterStageTask for type Approval. + // Remove waitTime from the updateRun status for BeforeStageTask and AfterStageTask for type Approval. removeWaitTimeFromUpdateRunStatus(updateRun) // Compute the update stages. @@ -344,6 +344,12 @@ func (r *Reconciler) computeRunStageStatus( // Apply the label selectors from the UpdateStrategy to filter the clusters. for _, stage := range updateRunStatus.UpdateStrategySnapshot.Stages { + if err := validateBeforeStageTask(stage.BeforeStageTasks); err != nil { + klog.ErrorS(err, "Failed to validate the before stage tasks", "updateStrategy", strategyKey, "stageName", stage.Name, "updateRun", updateRunRef) + // no more retries here. + invalidBeforeStageErr := controller.NewUserError(fmt.Errorf("the before stage tasks are invalid, updateStrategy: `%s`, stage: %s, err: %s", strategyKey, stage.Name, err.Error())) + return fmt.Errorf("%w: %s", errInitializedFailed, invalidBeforeStageErr.Error()) + } if err := validateAfterStageTask(stage.AfterStageTasks); err != nil { klog.ErrorS(err, "Failed to validate the after stage tasks", "updateStrategy", strategyKey, "stageName", stage.Name, "updateRun", updateRunRef) // no more retries here. @@ -418,12 +424,20 @@ func (r *Reconciler) computeRunStageStatus( curStageUpdatingStatus.Clusters[i].ClusterName = cluster.Name } + // Create the before stage tasks. + curStageUpdatingStatus.BeforeStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.BeforeStageTasks)) + for i, task := range stage.BeforeStageTasks { + curStageUpdatingStatus.BeforeStageTaskStatus[i].Type = task.Type + if task.Type == placementv1beta1.StageTaskTypeApproval { + curStageUpdatingStatus.BeforeStageTaskStatus[i].ApprovalRequestName = fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + } // Create the after stage tasks. curStageUpdatingStatus.AfterStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.AfterStageTasks)) for i, task := range stage.AfterStageTasks { curStageUpdatingStatus.AfterStageTaskStatus[i].Type = task.Type if task.Type == placementv1beta1.StageTaskTypeApproval { - curStageUpdatingStatus.AfterStageTaskStatus[i].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + curStageUpdatingStatus.AfterStageTaskStatus[i].ApprovalRequestName = fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) } } stagesStatus = append(stagesStatus, curStageUpdatingStatus) @@ -448,8 +462,25 @@ func (r *Reconciler) computeRunStageStatus( return nil } -// validateAfterStageTask valides the afterStageTasks in the stage defined in the UpdateStrategy. -// The error returned from this function is not retryable. +// validateBeforeStageTask validates the beforeStageTasks in the stage defined in the UpdateStrategy. +// The error returned from this function is not retriable. +func validateBeforeStageTask(tasks []placementv1beta1.StageTask) error { + if len(tasks) > 1 { + return fmt.Errorf("beforeStageTasks can have at most one task") + } + for i, task := range tasks { + if task.Type != placementv1beta1.StageTaskTypeApproval { + return fmt.Errorf("task %d of type %s is not allowed in beforeStageTasks, allowed type: Approval", i, task.Type) + } + if task.WaitTime != nil { + return fmt.Errorf("task %d of type Approval cannot have wait duration set", i) + } + } + return nil +} + +// validateAfterStageTask validates the afterStageTasks in the stage defined in the UpdateStrategy. +// The error returned from this function is not retriable. func validateAfterStageTask(tasks []placementv1beta1.StageTask) error { if len(tasks) == 2 && tasks[0].Type == tasks[1].Type { return fmt.Errorf("afterStageTasks cannot have two tasks of the same type: %s", tasks[0].Type) @@ -588,7 +619,7 @@ func (r *Reconciler) recordInitializationSucceeded(ctx context.Context, updateRu Status: metav1.ConditionTrue, ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunInitializeSucceededReason, - Message: "ClusterStagedUpdateRun initialized successfully", + Message: "The UpdateRun initialized successfully", }) if updateErr := r.Client.Status().Update(ctx, updateRun); updateErr != nil { klog.ErrorS(updateErr, "Failed to update the UpdateRun status as initialized", "updateRun", klog.KObj(updateRun)) diff --git a/pkg/controllers/updaterun/initialization_integration_test.go b/pkg/controllers/updaterun/initialization_integration_test.go index 8c5cebd59..5948ea6fd 100644 --- a/pkg/controllers/updaterun/initialization_integration_test.go +++ b/pkg/controllers/updaterun/initialization_integration_test.go @@ -874,14 +874,14 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the clusterStagedUpdateRun stats") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - want := generateExecutionStartedStatus(updateRun, initialized) + want := generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") By("Validating the clusterStagedUpdateRun initialized consistently") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, want, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) }) It("Should put related ClusterResourceOverrides in the status", func() { @@ -896,14 +896,14 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the clusterStagedUpdateRun stats") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - want := generateExecutionStartedStatus(updateRun, initialized) + want := generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") By("Validating the clusterStagedUpdateRun initialized consistently") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, want, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) }) It("Should pick latest master resource snapshot if multiple snapshots", func() { @@ -931,14 +931,14 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the clusterStagedUpdateRun status") initialized := generateSucceededInitializationStatus(crp, updateRun, "2", policySnapshot, updateStrategy, clusterResourceOverride) - want := generateExecutionStartedStatus(updateRun, initialized) + want := generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") By("Validating the clusterStagedUpdateRun initialized consistently") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, want, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) }) }) }) @@ -1010,15 +1010,25 @@ func generateSucceededInitializationStatus( }, } for i := range status.StagesStatus { - var tasks []placementv1beta1.StageTaskStatus + var beforeTasks []placementv1beta1.StageTaskStatus + for _, task := range updateStrategy.Spec.Stages[i].BeforeStageTasks { + taskStatus := placementv1beta1.StageTaskStatus{Type: task.Type} + if task.Type == placementv1beta1.StageTaskTypeApproval { + taskStatus.ApprovalRequestName = fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, updateRun.Name, status.StagesStatus[i].StageName) + } + beforeTasks = append(beforeTasks, taskStatus) + } + status.StagesStatus[i].BeforeStageTaskStatus = beforeTasks + + var afterTasks []placementv1beta1.StageTaskStatus for _, task := range updateStrategy.Spec.Stages[i].AfterStageTasks { taskStatus := placementv1beta1.StageTaskStatus{Type: task.Type} if task.Type == placementv1beta1.StageTaskTypeApproval { - taskStatus.ApprovalRequestName = updateRun.Name + "-" + status.StagesStatus[i].StageName + taskStatus.ApprovalRequestName = fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, updateRun.Name, status.StagesStatus[i].StageName) } - tasks = append(tasks, taskStatus) + afterTasks = append(afterTasks, taskStatus) } - status.StagesStatus[i].AfterStageTaskStatus = tasks + status.StagesStatus[i].AfterStageTaskStatus = afterTasks } return status } @@ -1056,28 +1066,56 @@ func generateSucceededInitializationStatusForSmallClusters( }, } for i := range status.StagesStatus { - var tasks []placementv1beta1.StageTaskStatus + var beforeTasks []placementv1beta1.StageTaskStatus + for _, task := range updateStrategy.Spec.Stages[i].BeforeStageTasks { + taskStatus := placementv1beta1.StageTaskStatus{Type: task.Type} + if task.Type == placementv1beta1.StageTaskTypeApproval { + taskStatus.ApprovalRequestName = fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, updateRun.Name, status.StagesStatus[i].StageName) + } + beforeTasks = append(beforeTasks, taskStatus) + } + status.StagesStatus[i].BeforeStageTaskStatus = beforeTasks + + var afterTasks []placementv1beta1.StageTaskStatus for _, task := range updateStrategy.Spec.Stages[i].AfterStageTasks { taskStatus := placementv1beta1.StageTaskStatus{Type: task.Type} if task.Type == placementv1beta1.StageTaskTypeApproval { - taskStatus.ApprovalRequestName = updateRun.Name + "-" + status.StagesStatus[i].StageName + taskStatus.ApprovalRequestName = fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, updateRun.Name, status.StagesStatus[i].StageName) } - tasks = append(tasks, taskStatus) + afterTasks = append(afterTasks, taskStatus) } - status.StagesStatus[i].AfterStageTaskStatus = tasks + status.StagesStatus[i].AfterStageTaskStatus = afterTasks } return status } func generateExecutionStartedStatus( updateRun *placementv1beta1.ClusterStagedUpdateRun, - initialized *placementv1beta1.UpdateRunStatus, + status *placementv1beta1.UpdateRunStatus, ) *placementv1beta1.UpdateRunStatus { // Mark updateRun execution has started. - initialized.Conditions = append(initialized.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + meta.SetStatusCondition(&status.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + // Mark updateRun 1st stage has started. - initialized.StagesStatus[0].Conditions = append(initialized.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + meta.SetStatusCondition(&status.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + // Mark updateRun 1st cluster in the 1st stage has started. - initialized.StagesStatus[0].Clusters[0].Conditions = []metav1.Condition{generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)} - return initialized + status.StagesStatus[0].Clusters[0].Conditions = []metav1.Condition{generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)} + return status +} + +func generateExecutionNotStartedStatus( + updateRun *placementv1beta1.ClusterStagedUpdateRun, + status *placementv1beta1.UpdateRunStatus, +) *placementv1beta1.UpdateRunStatus { + // Mark updateRun execution has not started. + status.Conditions = append(status.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + + // Mark updateRun 1st stage has not started. + status.StagesStatus[0].Conditions = append(status.StagesStatus[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + + // Mark updateRun 1st stage BeforeStageTasks has created approval request. + status.StagesStatus[0].BeforeStageTaskStatus[0].Conditions = append(status.StagesStatus[0].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) + return status } diff --git a/pkg/controllers/updaterun/initialization_test.go b/pkg/controllers/updaterun/initialization_test.go index f7ab0b2cd..446f70de7 100644 --- a/pkg/controllers/updaterun/initialization_test.go +++ b/pkg/controllers/updaterun/initialization_test.go @@ -1,30 +1,114 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package updaterun import ( + "fmt" "testing" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" - "go.goms.io/fleet/apis/placement/v1beta1" + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" ) +func TestValidateBeforeStageTask(t *testing.T) { + tests := []struct { + name string + task []placementv1beta1.StageTask + wantErr bool + wantErrMsg string + }{ + { + name: "valid BeforeTasks", + task: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + wantErr: false, + }, + { + name: "invalid BeforeTasks, greater than 1 task", + task: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, + wantErr: true, + wantErrMsg: "beforeStageTasks can have at most one task", + }, + { + name: "invalid BeforeTasks, with invalid task type", + task: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeTimedWait, + WaitTime: ptr.To(metav1.Duration{Duration: 5 * time.Minute}), + }, + }, + wantErr: true, + wantErrMsg: fmt.Sprintf("task %d of type %s is not allowed in beforeStageTasks, allowed type: Approval", 0, placementv1beta1.StageTaskTypeTimedWait), + }, + { + name: "invalid BeforeTasks, with duration for Approval", + task: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + WaitTime: ptr.To(metav1.Duration{Duration: 1 * time.Minute}), + }, + }, + wantErr: true, + wantErrMsg: fmt.Sprintf("task %d of type Approval cannot have wait duration set", 0), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotErr := validateBeforeStageTask(tt.task) + if tt.wantErr { + if gotErr == nil || gotErr.Error() != tt.wantErrMsg { + t.Fatalf("validateBeforeStageTask() error = %v, wantErr %v", gotErr, tt.wantErrMsg) + } + } else if gotErr != nil { + t.Fatalf("validateBeforeStageTask() error = %v, wantErr %v", gotErr, tt.wantErr) + } + }) + } +} + func TestValidateAfterStageTask(t *testing.T) { tests := []struct { name string - task []v1beta1.StageTask + task []placementv1beta1.StageTask wantErr bool errMsg string }{ { name: "valid AfterTasks", - task: []v1beta1.StageTask{ + task: []placementv1beta1.StageTask{ { - Type: v1beta1.StageTaskTypeApproval, + Type: placementv1beta1.StageTaskTypeApproval, }, { - Type: v1beta1.StageTaskTypeTimedWait, + Type: placementv1beta1.StageTaskTypeTimedWait, WaitTime: ptr.To(metav1.Duration{Duration: 5 * time.Minute}), }, }, @@ -32,13 +116,13 @@ func TestValidateAfterStageTask(t *testing.T) { }, { name: "invalid AfterTasks, same type of tasks", - task: []v1beta1.StageTask{ + task: []placementv1beta1.StageTask{ { - Type: v1beta1.StageTaskTypeTimedWait, + Type: placementv1beta1.StageTaskTypeTimedWait, WaitTime: ptr.To(metav1.Duration{Duration: 1 * time.Minute}), }, { - Type: v1beta1.StageTaskTypeTimedWait, + Type: placementv1beta1.StageTaskTypeTimedWait, WaitTime: ptr.To(metav1.Duration{Duration: 5 * time.Minute}), }, }, @@ -47,9 +131,9 @@ func TestValidateAfterStageTask(t *testing.T) { }, { name: "invalid AfterTasks, with nil duration for TimedWait", - task: []v1beta1.StageTask{ + task: []placementv1beta1.StageTask{ { - Type: v1beta1.StageTaskTypeTimedWait, + Type: placementv1beta1.StageTaskTypeTimedWait, }, }, wantErr: true, @@ -57,9 +141,9 @@ func TestValidateAfterStageTask(t *testing.T) { }, { name: "invalid AfterTasks, with zero duration for TimedWait", - task: []v1beta1.StageTask{ + task: []placementv1beta1.StageTask{ { - Type: v1beta1.StageTaskTypeTimedWait, + Type: placementv1beta1.StageTaskTypeTimedWait, WaitTime: ptr.To(metav1.Duration{Duration: 0 * time.Minute}), }, }, diff --git a/pkg/controllers/updaterun/validation_integration_test.go b/pkg/controllers/updaterun/validation_integration_test.go index 85ad26af9..0d37e7ce2 100644 --- a/pkg/controllers/updaterun/validation_integration_test.go +++ b/pkg/controllers/updaterun/validation_integration_test.go @@ -34,6 +34,7 @@ import ( clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" "go.goms.io/fleet/pkg/utils" + "go.goms.io/fleet/pkg/utils/condition" ) var _ = Describe("UpdateRun validation tests", func() { @@ -111,7 +112,7 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the initialization succeeded") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - wantStatus = generateExecutionStartedStatus(updateRun, initialized) + wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) @@ -164,7 +165,7 @@ var _ = Describe("UpdateRun validation tests", func() { Context("Test validateCRP", func() { AfterEach(func() { By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateFailedMetric(updateRun)) }) It("Should fail to validate if the CRP is not found", func() { @@ -208,7 +209,7 @@ var _ = Describe("UpdateRun validation tests", func() { validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "no latest policy snapshot associated") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateFailedMetric(updateRun)) }) It("Should fail to validate if the latest policySnapshot has changed", func() { @@ -237,7 +238,7 @@ var _ = Describe("UpdateRun validation tests", func() { Expect(k8sClient.Delete(ctx, newPolicySnapshot)).Should(Succeed()) By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateFailedMetric(updateRun)) }) It("Should fail to validate if the cluster count has changed", func() { @@ -251,14 +252,14 @@ var _ = Describe("UpdateRun validation tests", func() { "the cluster count initialized in the updateRun is outdated") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateFailedMetric(updateRun)) }) }) Context("Test validateStagesStatus", func() { AfterEach(func() { By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateFailedMetric(updateRun)) }) It("Should fail to validate if the UpdateStrategySnapshot is nil", func() { @@ -443,7 +444,7 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the initialization succeeded") initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - wantStatus = generateExecutionStartedStatus(updateRun, initialized) + wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) @@ -507,7 +508,7 @@ var _ = Describe("UpdateRun validation tests", func() { validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, wantStatus, "") By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun)) }) }) }) @@ -564,7 +565,7 @@ func generateFailedValidationStatus( updateRun *placementv1beta1.ClusterStagedUpdateRun, started *placementv1beta1.UpdateRunStatus, ) *placementv1beta1.UpdateRunStatus { - started.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, false) + started.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunFailedReason) started.Conditions = append(started.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) return started } diff --git a/pkg/controllers/workapplier/status.go b/pkg/controllers/workapplier/status.go index 75d7f1d6f..a309dbea5 100644 --- a/pkg/controllers/workapplier/status.go +++ b/pkg/controllers/workapplier/status.go @@ -28,9 +28,17 @@ import ( fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" "go.goms.io/fleet/pkg/utils/condition" "go.goms.io/fleet/pkg/utils/controller" + "go.goms.io/fleet/pkg/utils/resource" +) + +const ( + WorkStatusTrimmedDueToOversizedStatusReason = "Oversized" + WorkStatusTrimmedDueToOversizedStatusMsgTmpl = "The status data (drift/diff details and back-reported status) has been trimmed due to size constraints (%d bytes over limit %d)" ) // refreshWorkStatus refreshes the status of a Work object based on the processing results of its manifests. +// +// TO-DO (chenyu1): refactor this method a bit to reduce its complexity and enable parallelization. func (r *Reconciler) refreshWorkStatus( ctx context.Context, work *fleetv1beta1.Work, @@ -184,6 +192,47 @@ func (r *Reconciler) refreshWorkStatus( setWorkDiffReportedCondition(work, manifestCount, diffReportedObjectsCount) work.Status.ManifestConditions = rebuiltManifestConds + // Perform a size check before the status update. If the Work object goes over the size limit, trim + // some data from its status to ensure that update ops can go through. + // + // Note (chenyu1): at this moment, for simplicity reasons, the trimming op follows a very simple logic: + // if the size limit is breached, the work applier will summarize all drift/diff details in the status, + // and drop all back-reported status data. More sophisticated trimming logic does obviously exist; here + // the controller prefers the simple version primarily for two reasons: + // + // a) in most of the time, it is rare to reach the size limit: KubeFleet's snapshotting mechanism + // tries to keep the total manifest size in a Work object below 800KB (exceptions do exist), which leaves ~600KB + // space for the status data. The work applier reports for each manifest two conditions at most in the + // status (which are all quite small in size), plus the drift/diff details and the back-reported status + // (if applicable); considering the observation that drifts/diffs are not common and their details are usually small + // (just a JSON path plus the before/after values), and the observation that most Kubernetes objects + // only have a few KBs of status data and not all API types need status back-reporting, most of the time + // the Work object should have enough space for status data without trimming; + // b) performing more fine-grained, selective trimming can be a very CPU and memory intensive (e.g. + // various serialization calls) and complex process, and it is difficult to yield optimal results + // even with best efforts. + // + // TO-DO (chenyu1): re-visit this part of the code and evaluate the need for more fine-grained sharding + // if we have users that do use placements of a large collection of manifests and/or very large objects + // with drift/diff detection and status back-reporting on. + // + // TO-DO (chenyu1): evaluate if we need to impose more strict size limits on the manifests to ensure that + // Work objects (almost) always have enough space for status data. + sizeDeltaBytes, err := resource.CalculateSizeDeltaOverLimitFor(work, resource.DefaultObjSizeLimitWithPaddingBytes) + if err != nil { + // Normally this should never occur. + klog.ErrorS(err, "Failed to check Work object size before status update", "work", klog.KObj(work)) + wrappedErr := fmt.Errorf("failed to check work object size before status update: %w", err) + return controller.NewUnexpectedBehaviorError(wrappedErr) + } + if sizeDeltaBytes > 0 { + klog.V(2).InfoS("Must trim status data as the work object has grown over its size limit", + "work", klog.KObj(work), + "sizeDeltaBytes", sizeDeltaBytes, "sizeLimitBytes", resource.DefaultObjSizeLimitWithPaddingBytes) + trimWorkStatusDataWhenOversized(work) + } + setWorkStatusTrimmedCondition(work, sizeDeltaBytes, resource.DefaultObjSizeLimitWithPaddingBytes) + // Update the Work object status. if err := r.hubClient.Status().Update(ctx, work); err != nil { return controller.NewAPIServerError(false, err) @@ -643,3 +692,78 @@ func prepareRebuiltManifestCondQIdx(bundles []*manifestProcessingBundle) map[str } return rebuiltManifestCondQIdx } + +// trimWorkStatusDataWhenOversized trims some data from the Work object status when the object +// reaches its size limit. +func trimWorkStatusDataWhenOversized(work *fleetv1beta1.Work) { + // Trim drift/diff details + back-reported status from the Work object status. + // Replace detailed reportings with a summary if applicable. + for idx := range work.Status.ManifestConditions { + manifestCond := &work.Status.ManifestConditions[idx] + + // Note (chenyu1): check for the second term will always pass; it is added as a sanity check. + if manifestCond.DriftDetails != nil && len(manifestCond.DriftDetails.ObservedDrifts) > 0 { + driftCount := len(manifestCond.DriftDetails.ObservedDrifts) + firstDriftPath := manifestCond.DriftDetails.ObservedDrifts[0].Path + // If there are multiple drifts, report only the path of the first drift plus the count of + // other paths. Also, leave out the specific value differences. + pathSummary := firstDriftPath + if len(manifestCond.DriftDetails.ObservedDrifts) > 1 { + pathSummary = fmt.Sprintf("%s and %d more path(s)", firstDriftPath, driftCount-1) + } + manifestCond.DriftDetails.ObservedDrifts = []fleetv1beta1.PatchDetail{ + { + Path: pathSummary, + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + } + } + + // Note (chenyu1): check for the second term will always pass; it is added as a sanity check. + if manifestCond.DiffDetails != nil && len(manifestCond.DiffDetails.ObservedDiffs) > 0 { + diffCount := len(manifestCond.DiffDetails.ObservedDiffs) + firstDiffPath := manifestCond.DiffDetails.ObservedDiffs[0].Path + // If there are multiple drifts, report only the path of the first drift plus the count of + // other paths. Also, leave out the specific value differences. + pathSummary := firstDiffPath + if len(manifestCond.DiffDetails.ObservedDiffs) > 1 { + pathSummary = fmt.Sprintf("%s and %d more path(s)", firstDiffPath, diffCount-1) + } + manifestCond.DiffDetails.ObservedDiffs = []fleetv1beta1.PatchDetail{ + { + Path: pathSummary, + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + } + } + + manifestCond.BackReportedStatus = nil + } +} + +// setWorkStatusTrimmedCondition sets or removes the StatusTrimmed condition on a Work object +// based on whether the status has been trimmed due to it being oversized. +// +// Note (chenyu1): at this moment, due to limitations on the hub agent controller side (some +// controllers assume that placement related conditions are always set in a specific sequence), +// this StatusTrimmed condition might not be exposed on the placement status properly yet. +func setWorkStatusTrimmedCondition(work *fleetv1beta1.Work, sizeDeltaBytes, sizeLimitBytes int) { + if sizeDeltaBytes <= 0 { + // Drop the StatusTrimmed condition if it exists. + if isCondRemoved := meta.RemoveStatusCondition(&work.Status.Conditions, fleetv1beta1.WorkConditionTypeStatusTrimmed); isCondRemoved { + klog.V(2).InfoS("StatusTrimmed condition removed from Work object status", "work", klog.KObj(work)) + } + return + } + + // Set or update the StatusTrimmed condition. + meta.SetStatusCondition(&work.Status.Conditions, metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeStatusTrimmed, + Status: metav1.ConditionTrue, + Reason: WorkStatusTrimmedDueToOversizedStatusReason, + Message: fmt.Sprintf(WorkStatusTrimmedDueToOversizedStatusMsgTmpl, sizeDeltaBytes, sizeLimitBytes), + ObservedGeneration: work.Generation, + }) +} diff --git a/pkg/controllers/workapplier/status_test.go b/pkg/controllers/workapplier/status_test.go index 88a9adf8d..77744675e 100644 --- a/pkg/controllers/workapplier/status_test.go +++ b/pkg/controllers/workapplier/status_test.go @@ -26,6 +26,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" @@ -2030,3 +2031,721 @@ func TestSetWorkDiffReportedCondition(t *testing.T) { }) } } + +// TestTrimWorkStatusDataWhenOversized tests the trimWorkStatusDataWhenOversized function. +func TestTrimWorkStatusDataWhenOversized(t *testing.T) { + now := metav1.Now() + + testCases := []struct { + name string + work *fleetv1beta1.Work + wantWorkStatus fleetv1beta1.WorkStatus + }{ + { + name: "no drift/diff data, no back-reported status", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAvailableReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAvailableReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + }, + }, + }, + }, + }, + }, + { + name: "trim drifts (single drift)", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservationTime: now, + FirstDriftedObservedTime: now, + ObservedInMemberClusterGeneration: 1, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservationTime: now, + FirstDriftedObservedTime: now, + ObservedInMemberClusterGeneration: 1, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + }, + }, + }, + }, + }, + }, + { + name: "trim drifts (multiple drifts)", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservationTime: now, + FirstDriftedObservedTime: now, + ObservedInMemberClusterGeneration: 1, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s-1", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + { + Path: fmt.Sprintf("/metadata/labels/%s-2", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservationTime: now, + FirstDriftedObservedTime: now, + ObservedInMemberClusterGeneration: 1, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s-1 and %d more path(s)", dummyLabelKey, 1), + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + }, + }, + }, + }, + }, + }, + { + name: "trim diffs (single diff)", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservationTime: now, + FirstDiffedObservedTime: now, + ObservedInMemberClusterGeneration: ptr.To(int64(1)), + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservationTime: now, + FirstDiffedObservedTime: now, + ObservedInMemberClusterGeneration: ptr.To(int64(1)), + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + }, + }, + }, + }, + }, + }, + { + name: "trim diffs (multiple diffs)", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservationTime: now, + FirstDiffedObservedTime: now, + ObservedInMemberClusterGeneration: ptr.To(int64(1)), + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s-1", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + { + Path: fmt.Sprintf("/metadata/labels/%s-2", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservationTime: now, + FirstDiffedObservedTime: now, + ObservedInMemberClusterGeneration: ptr.To(int64(1)), + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s-1 and %d more path(s)", dummyLabelKey, 1), + ValueInMember: "(omitted)", + ValueInHub: "(omitted)", + }, + }, + }, + }, + }, + }, + }, + { + name: "trim back-reported status", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Namespace: memberReservedNSName1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAvailableReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + }, + }, + BackReportedStatus: &fleetv1beta1.BackReportedStatus{ + ObservationTime: now, + ObservedStatus: runtime.RawExtension{ + Raw: []byte(dummyLabelValue1), + }, + }, + }, + }, + }, + }, + wantWorkStatus: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAvailableReason, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Namespace: deployName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + }, + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + trimWorkStatusDataWhenOversized(tc.work) + if diff := cmp.Diff(tc.work.Status, tc.wantWorkStatus); diff != "" { + t.Errorf("trimmed work status mismatches (-got, +want):\n%s", diff) + } + }) + } +} + +// TestSetWorkStatusTrimmedCondition tests the setWorkStatusTrimmedCondition function. +func TestSetWorkStatusTrimmedCondition(t *testing.T) { + testCases := []struct { + name string + work *fleetv1beta1.Work + sizeDeltaBytes int + sizeLimitBytes int + wantWorkStatusConditions []metav1.Condition + }{ + { + name: "no trimming needed (size delta <= 0)", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Generation: 1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + }, + }, + }, + sizeDeltaBytes: 0, + sizeLimitBytes: 1024, + wantWorkStatusConditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + }, + }, + { + name: "remove existing StatusTrimmed condition when size delta <= 0", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Generation: 2, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 2, + }, + { + Type: fleetv1beta1.WorkConditionTypeStatusTrimmed, + Status: metav1.ConditionTrue, + Reason: WorkStatusTrimmedDueToOversizedStatusReason, + Message: fmt.Sprintf(WorkStatusTrimmedDueToOversizedStatusMsgTmpl, 500, 1024), + ObservedGeneration: 1, + }, + }, + }, + }, + sizeDeltaBytes: 0, + sizeLimitBytes: 1024, + wantWorkStatusConditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 2, + }, + }, + }, + { + name: "set StatusTrimmed condition when size delta > 0", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Generation: 1, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + }, + }, + }, + sizeDeltaBytes: 500, + sizeLimitBytes: 1024, + wantWorkStatusConditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + { + Type: fleetv1beta1.WorkConditionTypeStatusTrimmed, + Status: metav1.ConditionTrue, + Reason: WorkStatusTrimmedDueToOversizedStatusReason, + Message: fmt.Sprintf(WorkStatusTrimmedDueToOversizedStatusMsgTmpl, 500, 1024), + ObservedGeneration: 1, + }, + }, + }, + { + name: "update existing StatusTrimmed condition with new values", + work: &fleetv1beta1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: workName, + Generation: 2, + }, + Status: fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 2, + }, + { + Type: fleetv1beta1.WorkConditionTypeStatusTrimmed, + Status: metav1.ConditionTrue, + Reason: WorkStatusTrimmedDueToOversizedStatusReason, + Message: fmt.Sprintf(WorkStatusTrimmedDueToOversizedStatusMsgTmpl, 200, 1024), + ObservedGeneration: 1, + }, + }, + }, + }, + sizeDeltaBytes: 750, + sizeLimitBytes: 2048, + wantWorkStatusConditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 2, + }, + { + Type: fleetv1beta1.WorkConditionTypeStatusTrimmed, + Status: metav1.ConditionTrue, + Reason: WorkStatusTrimmedDueToOversizedStatusReason, + Message: fmt.Sprintf(WorkStatusTrimmedDueToOversizedStatusMsgTmpl, 750, 2048), + ObservedGeneration: 2, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + setWorkStatusTrimmedCondition(tc.work, tc.sizeDeltaBytes, tc.sizeLimitBytes) + if diff := cmp.Diff( + tc.work.Status.Conditions, tc.wantWorkStatusConditions, + ignoreFieldConditionLTTMsg, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("work status conditions mismatches (-got, +want):\n%s", diff) + } + }) + } +} diff --git a/pkg/metrics/hub/metrics.go b/pkg/metrics/hub/metrics.go index c61f85ff0..353cf99ea 100644 --- a/pkg/metrics/hub/metrics.go +++ b/pkg/metrics/hub/metrics.go @@ -23,16 +23,6 @@ import ( ) var ( - // These 2 metrics are used in v1alpha1 controller, should be soon deprecated. - PlacementApplyFailedCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "placement_apply_failed_counter", - Help: "Number of failed to apply cluster resource placement", - }, []string{"name"}) - PlacementApplySucceedCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "placement_apply_succeed_counter", - Help: "Number of successfully applied cluster resource placement", - }, []string{"name"}) - // FleetPlacementStatusLastTimeStampSeconds is a prometheus metric which keeps track of the last placement status. FleetPlacementStatusLastTimeStampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "fleet_workload_placement_status_last_timestamp_seconds", @@ -81,8 +71,6 @@ var ( func init() { metrics.Registry.MustRegister( - PlacementApplyFailedCount, - PlacementApplySucceedCount, FleetPlacementStatusLastTimeStampSeconds, FleetEvictionStatus, FleetUpdateRunStatusLastTimestampSeconds, diff --git a/pkg/resourcewatcher/change_dector.go b/pkg/resourcewatcher/change_dector.go index b3d89b24c..cb381d72c 100644 --- a/pkg/resourcewatcher/change_dector.go +++ b/pkg/resourcewatcher/change_dector.go @@ -84,6 +84,9 @@ type ChangeDetector struct { // ConcurrentResourceChangeWorker is the number of resource change work that are // allowed to sync concurrently. ConcurrentResourceChangeWorker int + + // EnableWorkload indicates whether workloads are allowed to run on the hub cluster. + EnableWorkload bool } // Start runs the detector, never stop until stopCh closed. This is called by the controller manager. @@ -189,7 +192,7 @@ func (d *ChangeDetector) dynamicResourceFilter(obj interface{}) bool { } if unstructuredObj, ok := obj.(*unstructured.Unstructured); ok { - shouldPropagate, err := utils.ShouldPropagateObj(d.InformerManager, unstructuredObj.DeepCopy()) + shouldPropagate, err := utils.ShouldPropagateObj(d.InformerManager, unstructuredObj.DeepCopy(), d.EnableWorkload) if err != nil || !shouldPropagate { klog.V(5).InfoS("Skip watching resource in namespace", "namespace", cwKey.Namespace, "group", cwKey.Group, "version", cwKey.Version, "kind", cwKey.Kind, "object", cwKey.Name) diff --git a/pkg/utils/common.go b/pkg/utils/common.go index 68f237ead..dfb367d01 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -505,29 +505,30 @@ func CheckCRDInstalled(discoveryClient discovery.DiscoveryInterface, gvk schema. return err } -// ShouldPropagateObj decides if one should propagate the object -func ShouldPropagateObj(informerManager informer.Manager, uObj *unstructured.Unstructured) (bool, error) { +// ShouldPropagateObj decides if one should propagate the object. +// PVCs are only propagated when enableWorkload is false (workloads not allowed on hub). +func ShouldPropagateObj(informerManager informer.Manager, uObj *unstructured.Unstructured, enableWorkload bool) (bool, error) { // TODO: add more special handling for different resource kind switch uObj.GroupVersionKind() { case appv1.SchemeGroupVersion.WithKind(ReplicaSetKind): - // Skip ReplicaSets if they are managed by Deployments (have owner references) - // Standalone ReplicaSets (without owners) can be propagated + // Skip ReplicaSets if they are managed by Deployments (have owner references). + // Standalone ReplicaSets (without owners) can be propagated. if len(uObj.GetOwnerReferences()) > 0 { return false, nil } case appv1.SchemeGroupVersion.WithKind("ControllerRevision"): - // Skip ControllerRevisions if they are managed by DaemonSets/StatefulSets (have owner references) - // These are automatically created by controllers and will be recreated on member clusters + // Skip ControllerRevisions if they are managed by DaemonSets/StatefulSets (have owner references). + // Standalone ControllerRevisions (without owners) can be propagated. if len(uObj.GetOwnerReferences()) > 0 { return false, nil } case corev1.SchemeGroupVersion.WithKind(ConfigMapKind): - // Skip the built-in custom CA certificate created in the namespace + // Skip the built-in custom CA certificate created in the namespace. if uObj.GetName() == "kube-root-ca.crt" { return false, nil } case corev1.SchemeGroupVersion.WithKind("ServiceAccount"): - // Skip the default service account created in the namespace + // Skip the default service account created in the namespace. if uObj.GetName() == "default" { return false, nil } @@ -541,6 +542,12 @@ func ShouldPropagateObj(informerManager informer.Manager, uObj *unstructured.Uns if secret.Type == corev1.SecretTypeServiceAccountToken { return false, nil } + case corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim"): + // Skip PersistentVolumeClaims by default to avoid conflicts with the PVCs created by statefulset controller. + // This only happens if the workloads are allowed to run on the hub cluster. + if enableWorkload { + return false, nil + } case corev1.SchemeGroupVersion.WithKind("Endpoints"): // we assume that all endpoints with the same name of a service is created by the service controller if _, err := informerManager.Lister(ServiceGVR).ByNamespace(uObj.GetNamespace()).Get(uObj.GetName()); err != nil { diff --git a/pkg/utils/common_test.go b/pkg/utils/common_test.go index 975ce8ed0..f41b900cc 100644 --- a/pkg/utils/common_test.go +++ b/pkg/utils/common_test.go @@ -1191,11 +1191,12 @@ func TestIsDiffedResourcePlacementEqual(t *testing.T) { } } -func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { +func TestShouldPropagateObj(t *testing.T) { tests := []struct { name string obj map[string]interface{} ownerReferences []metav1.OwnerReference + enableWorkload bool want bool }{ { @@ -1209,6 +1210,21 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { }, }, ownerReferences: nil, + enableWorkload: true, + want: true, + }, + { + name: "standalone replicaset without ownerReferences should propagate if workload is disabled", + obj: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "metadata": map[string]interface{}{ + "name": "standalone-rs", + "namespace": "default", + }, + }, + ownerReferences: nil, + enableWorkload: false, want: true, }, { @@ -1222,6 +1238,7 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { }, }, ownerReferences: nil, + enableWorkload: true, want: true, }, { @@ -1242,7 +1259,8 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { UID: "12345", }, }, - want: false, + enableWorkload: true, + want: false, }, { name: "pod owned by replicaset - passes ShouldPropagateObj but filtered by resource config", @@ -1262,7 +1280,8 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { UID: "67890", }, }, - want: true, // ShouldPropagateObj doesn't filter Pods - they're filtered by NewResourceConfig + enableWorkload: false, + want: true, // ShouldPropagateObj doesn't filter Pods - they're filtered by NewResourceConfig }, { name: "controllerrevision owned by daemonset should NOT propagate", @@ -1282,7 +1301,8 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { UID: "abcdef", }, }, - want: false, + enableWorkload: false, + want: false, }, { name: "controllerrevision owned by statefulset should NOT propagate", @@ -1302,7 +1322,8 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { UID: "fedcba", }, }, - want: false, + enableWorkload: false, + want: false, }, { name: "standalone controllerrevision without owner should propagate", @@ -1315,8 +1336,58 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { }, }, ownerReferences: nil, + enableWorkload: false, + want: true, + }, + { + name: "PVC should propagate when workload is disabled", + obj: map[string]interface{}{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": map[string]interface{}{ + "name": "test-pvc", + "namespace": "default", + }, + }, + ownerReferences: nil, + enableWorkload: false, want: true, }, + { + name: "PVC should NOT propagate when workload is enabled", + obj: map[string]interface{}{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": map[string]interface{}{ + "name": "test-pvc", + "namespace": "default", + }, + }, + ownerReferences: nil, + enableWorkload: true, + want: false, + }, + { + name: "PVC with ownerReferences should NOT propagate when workload is enabled", + obj: map[string]interface{}{ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": map[string]interface{}{ + "name": "data-statefulset-0", + "namespace": "default", + }, + }, + ownerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "statefulset", + UID: "sts-uid", + }, + }, + enableWorkload: true, + want: false, + }, } for _, tt := range tests { @@ -1326,7 +1397,7 @@ func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) { uObj.SetOwnerReferences(tt.ownerReferences) } - got, err := ShouldPropagateObj(nil, uObj) + got, err := ShouldPropagateObj(nil, uObj, tt.enableWorkload) if err != nil { t.Errorf("ShouldPropagateObj() error = %v", err) return diff --git a/pkg/utils/condition/reason.go b/pkg/utils/condition/reason.go index b1d963e58..9566ee42e 100644 --- a/pkg/utils/condition/reason.go +++ b/pkg/utils/condition/reason.go @@ -194,17 +194,20 @@ const ( // ClusterUpdatingSucceededReason is the reason string of condition if the cluster updating succeeded. ClusterUpdatingSucceededReason = "ClusterUpdatingSucceeded" - // AfterStageTaskApprovalRequestApprovedReason is the reason string of condition if the approval request for after stage task has been approved. - AfterStageTaskApprovalRequestApprovedReason = "AfterStageTaskApprovalRequestApproved" + // StageTaskApprovalRequestApprovedReason is the reason string of condition if the approval request for before or after stage task has been approved. + StageTaskApprovalRequestApprovedReason = "StageTaskApprovalRequestApproved" - // AfterStageTaskApprovalRequestCreatedReason is the reason string of condition if the approval request for after stage task has been created. - AfterStageTaskApprovalRequestCreatedReason = "AfterStageTaskApprovalRequestCreated" + // StageTaskApprovalRequestCreatedReason is the reason string of condition if the approval request for before or after stage task has been created. + StageTaskApprovalRequestCreatedReason = "StageTaskApprovalRequestCreated" // AfterStageTaskWaitTimeElapsedReason is the reason string of condition if the wait time for after stage task has elapsed. AfterStageTaskWaitTimeElapsedReason = "AfterStageTaskWaitTimeElapsed" // ApprovalRequestApprovalAcceptedReason is the reason string of condition if the approval of the approval request has been accepted. ApprovalRequestApprovalAcceptedReason = "ApprovalRequestApprovalAccepted" + + // UpdateRunWaitingMessageFmt is the message format string of condition if the staged update run is waiting for stage tasks in a stage to complete. + UpdateRunWaitingMessageFmt = "The updateRun is waiting for %s tasks in stage %s to complete" ) // A group of condition reason & message string which is used to populate the ClusterResourcePlacementEviction condition. diff --git a/pkg/utils/resource/resource.go b/pkg/utils/resource/resource.go index 0f514d8ab..405a08f97 100644 --- a/pkg/utils/resource/resource.go +++ b/pkg/utils/resource/resource.go @@ -21,6 +21,14 @@ import ( "crypto/sha256" "encoding/json" "fmt" + + "k8s.io/apimachinery/pkg/runtime" +) + +const ( + // etcd has a 1.5 MiB limit for objects by default, and Kubernetes clients might + // reject request entities too large (~2/~3 MiB, depending on the protocol in use). + DefaultObjSizeLimitWithPaddingBytes = 1415578 // 1.35 MiB, or ~1.42 MB. ) // HashOf returns the hash of the resource. @@ -31,3 +39,20 @@ func HashOf(resource any) (string, error) { } return fmt.Sprintf("%x", sha256.Sum256(jsonBytes)), nil } + +// CalculateSizeDeltaOverLimitFor calculates the size delta in bytes of a given object +// over a specified size limit. It returns a positive value if the object size exceeds +// the limit or a negative value if the object size is below the limit. +// +// This utility is useful in cases where KubeFleet needs to check if it can create/update +// an object with additional information. +func CalculateSizeDeltaOverLimitFor(obj runtime.Object, sizeLimitBytes int) (int, error) { + jsonBytes, err := json.Marshal(obj) + if err != nil { + return 0, fmt.Errorf("cannot determine object size: %w", err) + } + if sizeLimitBytes < 0 { + return 0, fmt.Errorf("size limit must be non-negative") + } + return len(jsonBytes) - sizeLimitBytes, nil +} diff --git a/pkg/utils/resource/resource_test.go b/pkg/utils/resource/resource_test.go index c3685b22a..526dac6e8 100644 --- a/pkg/utils/resource/resource_test.go +++ b/pkg/utils/resource/resource_test.go @@ -17,8 +17,13 @@ limitations under the License. package resource import ( + "encoding/json" "testing" + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" ) @@ -51,3 +56,65 @@ func TestHashOf(t *testing.T) { }) } } + +// TestCalculateSizeDeltaOverLimitFor tests the CalculateSizeDeltaOverLimitFor function. +func TestCalculateSizeDeltaOverLimitFor(t *testing.T) { + cm := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "app", + Namespace: "default", + }, + Data: map[string]string{ + "key": "value", + }, + } + cmBytes, err := json.Marshal(cm) + if err != nil { + t.Fatalf("Failed to marshal configMap") + } + cmSizeBytes := len(cmBytes) + + testCases := []struct { + name string + sizeLimitBytes int + wantErred bool + }{ + { + name: "under size limit (negative delta)", + sizeLimitBytes: 10000, + }, + { + name: "over size limit (positive delta)", + sizeLimitBytes: 1, + }, + { + name: "invalid size limit (negative size limit)", + // Invalid size limit. + sizeLimitBytes: -1, + wantErred: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + sizeDeltaBytes, err := CalculateSizeDeltaOverLimitFor(cm, tc.sizeLimitBytes) + + if tc.wantErred { + if err == nil { + t.Fatalf("CalculateSizeDeltaOverLimitFor() error = nil, want erred") + } + return + } + // Note: this test spec uses runtime calculation rather than static values for expected + // size delta comparison as different platforms have slight differences in the serialization process, + // which may produce different sizing results. + if !cmp.Equal(sizeDeltaBytes, cmSizeBytes-tc.sizeLimitBytes) { + t.Errorf("CalculateSizeDeltaOverLimitFor() = %d, want %d", sizeDeltaBytes, cmSizeBytes-tc.sizeLimitBytes) + } + }) + } +} diff --git a/pkg/webhook/add_handler.go b/pkg/webhook/add_handler.go index 1fa538be5..9cd262de5 100644 --- a/pkg/webhook/add_handler.go +++ b/pkg/webhook/add_handler.go @@ -16,13 +16,13 @@ import ( func init() { // AddToManagerFleetResourceValidator is a function to register fleet guard rail resource validator to the webhook server AddToManagerFleetResourceValidator = fleetresourcehandler.Add + AddToManagerMemberclusterValidator = membercluster.Add // AddToManagerFuncs is a list of functions to register webhook validators and mutators to the webhook server AddToManagerFuncs = append(AddToManagerFuncs, clusterresourceplacement.AddMutating) AddToManagerFuncs = append(AddToManagerFuncs, clusterresourceplacement.Add) AddToManagerFuncs = append(AddToManagerFuncs, resourceplacement.Add) AddToManagerFuncs = append(AddToManagerFuncs, pod.Add) AddToManagerFuncs = append(AddToManagerFuncs, replicaset.Add) - AddToManagerFuncs = append(AddToManagerFuncs, membercluster.Add) AddToManagerFuncs = append(AddToManagerFuncs, clusterresourceoverride.Add) AddToManagerFuncs = append(AddToManagerFuncs, resourceoverride.Add) AddToManagerFuncs = append(AddToManagerFuncs, clusterresourceplacementeviction.Add) diff --git a/pkg/webhook/clusterresourceoverride/clusterresourceoverride_validating_webhook.go b/pkg/webhook/clusterresourceoverride/clusterresourceoverride_validating_webhook.go index 47cc4aff5..709de919b 100644 --- a/pkg/webhook/clusterresourceoverride/clusterresourceoverride_validating_webhook.go +++ b/pkg/webhook/clusterresourceoverride/clusterresourceoverride_validating_webhook.go @@ -40,14 +40,16 @@ var ( ) type clusterResourceOverrideValidator struct { - client client.Client + // Note: we have to use the uncached client here to avoid getting stale data + // since we need to guarantee that a resource cannot be selected by multiple overrides. + client client.Reader decoder webhook.AdmissionDecoder } // Add registers the webhook for K8s bulit-in object types. func Add(mgr manager.Manager) error { hookServer := mgr.GetWebhookServer() - hookServer.Register(ValidationPath, &webhook.Admission{Handler: &clusterResourceOverrideValidator{mgr.GetClient(), admission.NewDecoder(mgr.GetScheme())}}) + hookServer.Register(ValidationPath, &webhook.Admission{Handler: &clusterResourceOverrideValidator{mgr.GetAPIReader(), admission.NewDecoder(mgr.GetScheme())}}) return nil } @@ -80,7 +82,7 @@ func (v *clusterResourceOverrideValidator) Handle(ctx context.Context, req admis } // listClusterResourceOverride returns a list of cluster resource overrides. -func listClusterResourceOverride(ctx context.Context, client client.Client) (*placementv1beta1.ClusterResourceOverrideList, error) { +func listClusterResourceOverride(ctx context.Context, client client.Reader) (*placementv1beta1.ClusterResourceOverrideList, error) { croList := &placementv1beta1.ClusterResourceOverrideList{} if err := client.List(ctx, croList); err != nil { klog.ErrorS(err, "Failed to list clusterResourceOverrides when validating") diff --git a/pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook_test.go b/pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook_test.go index 6c987f0da..5df4cd7ee 100644 --- a/pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook_test.go +++ b/pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook_test.go @@ -7,7 +7,7 @@ import ( "fmt" "testing" - "github.com/crossplane/crossplane-runtime/pkg/test" + "github.com/crossplane/crossplane-runtime/v2/pkg/test" "github.com/stretchr/testify/assert" admissionv1 "k8s.io/api/admission/v1" authenticationv1 "k8s.io/api/authentication/v1" diff --git a/pkg/webhook/membercluster/membercluster_validating_webhook.go b/pkg/webhook/membercluster/membercluster_validating_webhook.go index 4f9710d11..39d12b8dc 100644 --- a/pkg/webhook/membercluster/membercluster_validating_webhook.go +++ b/pkg/webhook/membercluster/membercluster_validating_webhook.go @@ -26,15 +26,19 @@ var ( ) type memberClusterValidator struct { - client client.Client - decoder webhook.AdmissionDecoder + client client.Client + decoder webhook.AdmissionDecoder + networkingAgentsEnabled bool } // Add registers the webhook for K8s bulit-in object types. -func Add(mgr manager.Manager) error { +func Add(mgr manager.Manager, networkingAgentsEnabled bool) { hookServer := mgr.GetWebhookServer() - hookServer.Register(ValidationPath, &webhook.Admission{Handler: &memberClusterValidator{client: mgr.GetClient(), decoder: admission.NewDecoder(mgr.GetScheme())}}) - return nil + hookServer.Register(ValidationPath, &webhook.Admission{Handler: &memberClusterValidator{ + client: mgr.GetClient(), + decoder: admission.NewDecoder(mgr.GetScheme()), + networkingAgentsEnabled: networkingAgentsEnabled, + }}) } // Handle memberClusterValidator checks to see if member cluster has valid fields. @@ -50,8 +54,12 @@ func (v *memberClusterValidator) Handle(ctx context.Context, req admission.Reque } if mc.Spec.DeleteOptions != nil && mc.Spec.DeleteOptions.ValidationMode == clusterv1beta1.DeleteValidationModeSkip { - klog.V(2).InfoS("Skipping validation for member cluster DELETE", "memberCluster", mcObjectName) - return admission.Allowed("Skipping validation for member cluster DELETE") + klog.V(2).InfoS("Skipping validation for member cluster DELETE when the validation mode is set to skip", "memberCluster", mcObjectName) + return admission.Allowed("Skipping validation for member cluster DELETE when the validation mode is set to skip") + } + if !v.networkingAgentsEnabled { + klog.V(2).InfoS("Networking agents disabled; skipping ServiceExport validation", "memberCluster", mcObjectName) + return admission.Allowed("Networking agents disabled; skipping ServiceExport validation") } klog.V(2).InfoS("Validating webhook member cluster DELETE", "memberCluster", mcObjectName) diff --git a/pkg/webhook/membercluster/membercluster_validating_webhook_test.go b/pkg/webhook/membercluster/membercluster_validating_webhook_test.go new file mode 100644 index 000000000..3cec42679 --- /dev/null +++ b/pkg/webhook/membercluster/membercluster_validating_webhook_test.go @@ -0,0 +1,141 @@ +package membercluster + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + + clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" + "go.goms.io/fleet/pkg/utils" + + fleetnetworkingv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +func TestHandleDelete(t *testing.T) { + t.Parallel() + + testCases := map[string]struct { + networkingEnabled bool + validationMode clusterv1beta1.DeleteValidationMode + wantAllowed bool + wantMessageSubstr string + }{ + "networking-disabled-allows-delete": { + networkingEnabled: false, + wantAllowed: true, + validationMode: clusterv1beta1.DeleteValidationModeStrict, + }, + "networking-enabled-denies-delete": { + networkingEnabled: true, + wantAllowed: false, + validationMode: clusterv1beta1.DeleteValidationModeStrict, + wantMessageSubstr: "Please delete serviceExport", + }, + "delete-options-skip-bypasses-validation": { + networkingEnabled: true, + wantAllowed: true, + validationMode: clusterv1beta1.DeleteValidationModeSkip, + }, + } + + for name, tc := range testCases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + mcName := fmt.Sprintf("member-%s", name) + namespaceName := fmt.Sprintf(utils.NamespaceNameFormat, mcName) + svcExport := newInternalServiceExport(mcName, namespaceName) + + validator := newMemberClusterValidatorForTest(t, tc.networkingEnabled, svcExport) + mc := &clusterv1beta1.MemberCluster{ObjectMeta: metav1.ObjectMeta{Name: mcName}} + mc.Spec.DeleteOptions = &clusterv1beta1.DeleteOptions{ValidationMode: tc.validationMode} + req := buildDeleteRequestFromObject(t, mc) + + resp := validator.Handle(context.Background(), req) + if resp.Allowed != tc.wantAllowed { + t.Fatalf("Handle() got response: %+v, want allowed %t", resp, tc.wantAllowed) + } + if tc.wantMessageSubstr != "" { + if resp.Result == nil || !strings.Contains(resp.Result.Message, tc.wantMessageSubstr) { + t.Fatalf("Handle() got response result: %v, want contain: %q", resp.Result, tc.wantMessageSubstr) + } + } + }) + } +} + +func newMemberClusterValidatorForTest(t *testing.T, networkingEnabled bool, objs ...client.Object) *memberClusterValidator { + t.Helper() + + scheme := runtime.NewScheme() + if err := clusterv1beta1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add member cluster scheme: %v", err) + } + if err := fleetnetworkingv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add fleet networking scheme: %v", err) + } + scheme.AddKnownTypes(fleetnetworkingv1alpha1.GroupVersion, + &fleetnetworkingv1alpha1.InternalServiceExport{}, + &fleetnetworkingv1alpha1.InternalServiceExportList{}, + ) + metav1.AddToGroupVersion(scheme, fleetnetworkingv1alpha1.GroupVersion) + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() + decoder := admission.NewDecoder(scheme) + + return &memberClusterValidator{ + client: fakeClient, + decoder: decoder, + networkingAgentsEnabled: networkingEnabled, + } +} + +func buildDeleteRequestFromObject(t *testing.T, mc *clusterv1beta1.MemberCluster) admission.Request { + t.Helper() + + raw, err := json.Marshal(mc) + if err != nil { + t.Fatalf("failed to marshal member cluster: %v", err) + } + + return admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + Name: mc.Name, + OldObject: runtime.RawExtension{Raw: raw}, + }, + } +} + +func newInternalServiceExport(clusterID, namespace string) *fleetnetworkingv1alpha1.InternalServiceExport { + return &fleetnetworkingv1alpha1.InternalServiceExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-service", + Namespace: namespace, + }, + Spec: fleetnetworkingv1alpha1.InternalServiceExportSpec{ + ServiceReference: fleetnetworkingv1alpha1.ExportedObjectReference{ + ClusterID: clusterID, + Kind: "Service", + Namespace: "work", + Name: "sample-service", + ResourceVersion: "1", + Generation: 1, + UID: types.UID("svc-uid"), + NamespacedName: "work/sample-service", + }, + }, + } +} diff --git a/pkg/webhook/resourceoverride/resourceoverride_validating_webhook.go b/pkg/webhook/resourceoverride/resourceoverride_validating_webhook.go index 149b794ed..06619cb81 100644 --- a/pkg/webhook/resourceoverride/resourceoverride_validating_webhook.go +++ b/pkg/webhook/resourceoverride/resourceoverride_validating_webhook.go @@ -40,14 +40,16 @@ var ( ) type resourceOverrideValidator struct { - client client.Client + // Note: we have to use the uncached client here to avoid getting stale data + // since we need to guarantee that a resource cannot be selected by multiple overrides. + client client.Reader decoder webhook.AdmissionDecoder } // Add registers the webhook for K8s bulit-in object types. func Add(mgr manager.Manager) error { hookServer := mgr.GetWebhookServer() - hookServer.Register(ValidationPath, &webhook.Admission{Handler: &resourceOverrideValidator{mgr.GetClient(), admission.NewDecoder(mgr.GetScheme())}}) + hookServer.Register(ValidationPath, &webhook.Admission{Handler: &resourceOverrideValidator{mgr.GetAPIReader(), admission.NewDecoder(mgr.GetScheme())}}) return nil } diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index e50e888f7..1dedb5cb1 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -131,14 +131,16 @@ var ( var AddToManagerFuncs []func(manager.Manager) error var AddToManagerFleetResourceValidator func(manager.Manager, []string, bool) error +var AddToManagerMemberclusterValidator func(manager.Manager, bool) // AddToManager adds all Controllers to the Manager -func AddToManager(m manager.Manager, whiteListedUsers []string, denyModifyMemberClusterLabels bool) error { +func AddToManager(m manager.Manager, whiteListedUsers []string, denyModifyMemberClusterLabels bool, networkingAgentsEnabled bool) error { for _, f := range AddToManagerFuncs { if err := f(m); err != nil { return err } } + AddToManagerMemberclusterValidator(m, networkingAgentsEnabled) return AddToManagerFleetResourceValidator(m, whiteListedUsers, denyModifyMemberClusterLabels) } diff --git a/test/apis/placement/v1beta1/api_validation_integration_test.go b/test/apis/placement/v1beta1/api_validation_integration_test.go index 1194ba210..6cf6d8c7e 100644 --- a/test/apis/placement/v1beta1/api_validation_integration_test.go +++ b/test/apis/placement/v1beta1/api_validation_integration_test.go @@ -1213,12 +1213,12 @@ var _ = Describe("Test placement v1beta1 API validation", func() { PlacementName: "test-placement", ResourceSnapshotIndex: "1", StagedUpdateStrategyName: "test-strategy", - State: placementv1beta1.StateNotStarted, + State: placementv1beta1.StateInitialize, }, } Expect(hubClient.Create(ctx, &updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateStarted + updateRun.Spec.State = placementv1beta1.StateRun Expect(hubClient.Update(ctx, &updateRun)).Should(Succeed()) Expect(hubClient.Delete(ctx, &updateRun)).Should(Succeed()) }) @@ -1823,7 +1823,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateNotStarted, + State: placementv1beta1.StateInitialize, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) @@ -1839,11 +1839,11 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: "unspecfied-state-update-run-" + fmt.Sprintf("%d", GinkgoParallelProcess()), }, Spec: placementv1beta1.UpdateRunSpec{ - // State not specified - should default to Initialize + // State not specified - should default to Initialize. }, } Expect(hubClient.Create(ctx, updateRunWithDefaultState)).Should(Succeed()) - Expect(updateRunWithDefaultState.Spec.State).To(Equal(placementv1beta1.StateNotStarted)) + Expect(updateRunWithDefaultState.Spec.State).To(Equal(placementv1beta1.StateInitialize)) Expect(hubClient.Delete(ctx, updateRunWithDefaultState)).Should(Succeed()) }) @@ -1857,22 +1857,17 @@ var _ = Describe("Test placement v1beta1 API validation", func() { }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - Expect(updateRun.Spec.State).To(Equal(placementv1beta1.StateNotStarted)) + Expect(updateRun.Spec.State).To(Equal(placementv1beta1.StateInitialize)) Expect(hubClient.Delete(ctx, updateRun)).Should(Succeed()) }) - It("should allow transition from Initialize to Execute", func() { - updateRun.Spec.State = placementv1beta1.StateStarted - Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) - }) - - It("should allow transition from Initialize to Abandon", func() { - updateRun.Spec.State = placementv1beta1.StateAbandoned + It("should allow transition from Initialize to Run", func() { + updateRun.Spec.State = placementv1beta1.StateRun Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) }) }) - Context("Test ClusterStagedUpdateRun State API validation - valid Execute state transitions", func() { + Context("Test ClusterStagedUpdateRun State API validation - valid Run state transitions", func() { var updateRun *placementv1beta1.ClusterStagedUpdateRun updateRunName := fmt.Sprintf(validupdateRunNameTemplate, GinkgoParallelProcess()) @@ -1882,7 +1877,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateStarted, + State: placementv1beta1.StateRun, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) @@ -1892,18 +1887,13 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Expect(hubClient.Delete(ctx, updateRun)).Should(Succeed()) }) - It("should allow transition from Execute to Pause", func() { - updateRun.Spec.State = placementv1beta1.StateStopped - Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) - }) - - It("should allow transition from Execute to Abandon", func() { - updateRun.Spec.State = placementv1beta1.StateAbandoned + It("should allow transition from Run to Stop", func() { + updateRun.Spec.State = placementv1beta1.StateStop Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) }) }) - Context("Test ClusterStagedUpdateRun State API validation - valid Pause state transitions", func() { + Context("Test ClusterStagedUpdateRun State API validation - valid Stop state transitions", func() { var updateRun *placementv1beta1.ClusterStagedUpdateRun updateRunName := fmt.Sprintf(validupdateRunNameTemplate, GinkgoParallelProcess()) @@ -1913,26 +1903,18 @@ var _ = Describe("Test placement v1beta1 API validation", func() { Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateStarted, + State: placementv1beta1.StateStop, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - // Transition to Pause state first - updateRun.Spec.State = placementv1beta1.StateStopped - Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) }) AfterEach(func() { Expect(hubClient.Delete(ctx, updateRun)).Should(Succeed()) }) - It("should allow transition from Pause to Execute", func() { - updateRun.Spec.State = placementv1beta1.StateStarted - Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) - }) - - It("should allow transition from Pause to Abandon", func() { - updateRun.Spec.State = placementv1beta1.StateAbandoned + It("should allow transition from Stop to Run", func() { + updateRun.Spec.State = placementv1beta1.StateRun Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) }) }) @@ -1947,117 +1929,59 @@ var _ = Describe("Test placement v1beta1 API validation", func() { } }) - It("should deny transition from Initialize to Pause", func() { - updateRun = &placementv1beta1.ClusterStagedUpdateRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: updateRunName, - }, - Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateNotStarted, - }, - } - Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - - updateRun.Spec.State = placementv1beta1.StateStopped - err := hubClient.Update(ctx, updateRun) - var statusErr *k8sErrors.StatusError - Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Initialize to Pause")) - }) - - It("should deny transition from Execute to Initialize", func() { - updateRun = &placementv1beta1.ClusterStagedUpdateRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: updateRunName, - }, - Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateStarted, - }, - } - Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - - updateRun.Spec.State = placementv1beta1.StateNotStarted - err := hubClient.Update(ctx, updateRun) - var statusErr *k8sErrors.StatusError - Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Execute to Initialize")) - }) - - It("should deny transition from Pause to Initialize", func() { - updateRun = &placementv1beta1.ClusterStagedUpdateRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: updateRunName, - }, - Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateStarted, - }, - } - Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - - // Transition to Pause first - updateRun.Spec.State = placementv1beta1.StateStopped - Expect(hubClient.Update(ctx, updateRun)).Should(Succeed()) - - // Try to transition back to Initialize - updateRun.Spec.State = placementv1beta1.StateNotStarted - err := hubClient.Update(ctx, updateRun) - var statusErr *k8sErrors.StatusError - Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Pause to Initialize")) - }) - - It("should deny transition from Abandon to Initialize", func() { + It("should deny transition from Initialize to Stop", func() { updateRun = &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateAbandoned, + State: placementv1beta1.StateInitialize, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateNotStarted + updateRun.Spec.State = placementv1beta1.StateStop err := hubClient.Update(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: Abandon is a terminal state and cannot transition to any other state")) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Initialize to Stop")) }) - It("should deny transition from Abandon to Execute", func() { + It("should deny transition from Run to Initialize", func() { updateRun = &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateAbandoned, + State: placementv1beta1.StateRun, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateStarted + updateRun.Spec.State = placementv1beta1.StateInitialize err := hubClient.Update(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: Abandon is a terminal state and cannot transition to any other state")) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Run to Initialize")) }) - It("should deny transition from Abandon to Pause", func() { + It("should deny transition from Stop to Initialize", func() { updateRun = &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ - State: placementv1beta1.StateAbandoned, + State: placementv1beta1.StateStop, }, } Expect(hubClient.Create(ctx, updateRun)).Should(Succeed()) - updateRun.Spec.State = placementv1beta1.StateStopped + // Try to transition back to Initialize. + updateRun.Spec.State = placementv1beta1.StateInitialize err := hubClient.Update(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Update ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: Abandon is a terminal state and cannot transition to any other state")) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("invalid state transition: cannot transition from Stop to Initialize")) }) }) @@ -2077,7 +2001,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { err := hubClient.Create(ctx, updateRun) var statusErr *k8sErrors.StatusError Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Create ClusterStagedUpdateRun call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&k8sErrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("supported values: \"Initialize\", \"Execute\", \"Pause\", \"Abandon\"")) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("supported values: \"Initialize\", \"Run\", \"Stop\"")) }) }) diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index 1315dc1e8..f36d7f105 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -2040,19 +2040,19 @@ func updateRunStageRolloutSucceedConditions(generation int64) []metav1.Condition } } -func updateRunAfterStageTaskSucceedConditions(generation int64, taskType placementv1beta1.StageTaskType) []metav1.Condition { +func updateRunStageTaskSucceedConditions(generation int64, taskType placementv1beta1.StageTaskType) []metav1.Condition { if taskType == placementv1beta1.StageTaskTypeApproval { return []metav1.Condition{ { Type: string(placementv1beta1.StageTaskConditionApprovalRequestCreated), Status: metav1.ConditionTrue, - Reason: condition.AfterStageTaskApprovalRequestCreatedReason, + Reason: condition.StageTaskApprovalRequestCreatedReason, ObservedGeneration: generation, }, { Type: string(placementv1beta1.StageTaskConditionApprovalRequestApproved), Status: metav1.ConditionTrue, - Reason: condition.AfterStageTaskApprovalRequestApprovedReason, + Reason: condition.StageTaskApprovalRequestApprovedReason, ObservedGeneration: generation, }, } @@ -2068,12 +2068,16 @@ func updateRunAfterStageTaskSucceedConditions(generation int64, taskType placeme } func updateRunSucceedConditions(generation int64) []metav1.Condition { + initializeCondGeneration := generation + if generation > 1 { + initializeCondGeneration = 1 + } return []metav1.Condition{ { Type: string(placementv1beta1.StagedUpdateRunConditionInitialized), Status: metav1.ConditionTrue, Reason: condition.UpdateRunInitializeSucceededReason, - ObservedGeneration: generation, + ObservedGeneration: initializeCondGeneration, }, { Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), @@ -2090,6 +2094,17 @@ func updateRunSucceedConditions(generation int64) []metav1.Condition { } } +func updateRunInitializedConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + { + Type: string(placementv1beta1.StagedUpdateRunConditionInitialized), + Status: metav1.ConditionTrue, + Reason: condition.UpdateRunInitializeSucceededReason, + ObservedGeneration: generation, + }, + } +} + func clusterStagedUpdateRunStatusSucceededActual( updateRunName string, wantResourceIndex string, @@ -2101,6 +2116,7 @@ func clusterStagedUpdateRunStatusSucceededActual( wantUnscheduledClusters []string, wantCROs map[string][]string, wantROs map[string][]placementv1beta1.NamespacedName, + execute bool, ) func() error { return func() error { updateRun := &placementv1beta1.ClusterStagedUpdateRun{} @@ -2116,9 +2132,15 @@ func clusterStagedUpdateRunStatusSucceededActual( UpdateStrategySnapshot: wantStrategySpec, } - wantStatus.StagesStatus = buildStageUpdatingStatuses(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) - wantStatus.DeletionStageStatus = buildDeletionStageStatus(wantUnscheduledClusters, updateRun) - wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation) + if execute { + wantStatus.StagesStatus = buildStageUpdatingStatuses(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) + wantStatus.DeletionStageStatus = buildDeletionStageStatus(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation) + } else { + wantStatus.StagesStatus = buildStageUpdatingStatusesForInitialized(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) + wantStatus.DeletionStageStatus = buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunInitializedConditions(updateRun.Generation) + } if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) } @@ -2136,6 +2158,7 @@ func stagedUpdateRunStatusSucceededActual( wantUnscheduledClusters []string, wantCROs map[string][]string, wantROs map[string][]placementv1beta1.NamespacedName, + execute bool, ) func() error { return func() error { updateRun := &placementv1beta1.StagedUpdateRun{} @@ -2151,9 +2174,15 @@ func stagedUpdateRunStatusSucceededActual( UpdateStrategySnapshot: wantStrategySpec, } - wantStatus.StagesStatus = buildStageUpdatingStatuses(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) - wantStatus.DeletionStageStatus = buildDeletionStageStatus(wantUnscheduledClusters, updateRun) - wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation) + if execute { + wantStatus.StagesStatus = buildStageUpdatingStatuses(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) + wantStatus.DeletionStageStatus = buildDeletionStageStatus(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation) + } else { + wantStatus.StagesStatus = buildStageUpdatingStatusesForInitialized(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) + wantStatus.DeletionStageStatus = buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunInitializedConditions(updateRun.Generation) + } if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) } @@ -2161,6 +2190,40 @@ func stagedUpdateRunStatusSucceededActual( } } +func buildStageUpdatingStatusesForInitialized( + wantStrategySpec *placementv1beta1.UpdateStrategySpec, + wantSelectedClusters [][]string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, + updateRun placementv1beta1.UpdateRunObj, +) []placementv1beta1.StageUpdatingStatus { + stagesStatus := make([]placementv1beta1.StageUpdatingStatus, len(wantStrategySpec.Stages)) + for i, stage := range wantStrategySpec.Stages { + stagesStatus[i].StageName = stage.Name + stagesStatus[i].Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantSelectedClusters[i])) + for j := range stagesStatus[i].Clusters { + stagesStatus[i].Clusters[j].ClusterName = wantSelectedClusters[i][j] + stagesStatus[i].Clusters[j].ClusterResourceOverrideSnapshots = wantCROs[wantSelectedClusters[i][j]] + stagesStatus[i].Clusters[j].ResourceOverrideSnapshots = wantROs[wantSelectedClusters[i][j]] + } + stagesStatus[i].BeforeStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.BeforeStageTasks)) + for j, task := range stage.BeforeStageTasks { + stagesStatus[i].BeforeStageTaskStatus[j].Type = task.Type + if task.Type == placementv1beta1.StageTaskTypeApproval { + stagesStatus[i].BeforeStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + } + stagesStatus[i].AfterStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.AfterStageTasks)) + for j, task := range stage.AfterStageTasks { + stagesStatus[i].AfterStageTaskStatus[j].Type = task.Type + if task.Type == placementv1beta1.StageTaskTypeApproval { + stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + } + } + return stagesStatus +} + func buildStageUpdatingStatuses( wantStrategySpec *placementv1beta1.UpdateStrategySpec, wantSelectedClusters [][]string, @@ -2178,13 +2241,21 @@ func buildStageUpdatingStatuses( stagesStatus[i].Clusters[j].ResourceOverrideSnapshots = wantROs[wantSelectedClusters[i][j]] stagesStatus[i].Clusters[j].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.GetGeneration()) } + stagesStatus[i].BeforeStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.BeforeStageTasks)) + for j, task := range stage.BeforeStageTasks { + stagesStatus[i].BeforeStageTaskStatus[j].Type = task.Type + if task.Type == placementv1beta1.StageTaskTypeApproval { + stagesStatus[i].BeforeStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.BeforeStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + stagesStatus[i].BeforeStageTaskStatus[j].Conditions = updateRunStageTaskSucceedConditions(updateRun.GetGeneration(), task.Type) + } stagesStatus[i].AfterStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.AfterStageTasks)) for j, task := range stage.AfterStageTasks { stagesStatus[i].AfterStageTaskStatus[j].Type = task.Type if task.Type == placementv1beta1.StageTaskTypeApproval { - stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.AfterStageApprovalTaskNameFmt, updateRun.GetName(), stage.Name) } - stagesStatus[i].AfterStageTaskStatus[j].Conditions = updateRunAfterStageTaskSucceedConditions(updateRun.GetGeneration(), task.Type) + stagesStatus[i].AfterStageTaskStatus[j].Conditions = updateRunStageTaskSucceedConditions(updateRun.GetGeneration(), task.Type) } stagesStatus[i].Conditions = updateRunStageRolloutSucceedConditions(updateRun.GetGeneration()) } @@ -2194,6 +2265,15 @@ func buildStageUpdatingStatuses( func buildDeletionStageStatus( wantUnscheduledClusters []string, updateRun placementv1beta1.UpdateRunObj, +) *placementv1beta1.StageUpdatingStatus { + deleteStageStatus := buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + deleteStageStatus.Conditions = updateRunStageRolloutSucceedConditions(updateRun.GetGeneration()) + return deleteStageStatus +} + +func buildDeletionStatusWithoutConditions( + wantUnscheduledClusters []string, + updateRun placementv1beta1.UpdateRunObj, ) *placementv1beta1.StageUpdatingStatus { deleteStageStatus := &placementv1beta1.StageUpdatingStatus{ StageName: "kubernetes-fleet.io/deleteStage", @@ -2203,7 +2283,6 @@ func buildDeletionStageStatus( deleteStageStatus.Clusters[i].ClusterName = wantUnscheduledClusters[i] deleteStageStatus.Clusters[i].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.GetGeneration()) } - deleteStageStatus.Conditions = updateRunStageRolloutSucceedConditions(updateRun.GetGeneration()) return deleteStageStatus } diff --git a/test/e2e/cluster_staged_updaterun_test.go b/test/e2e/cluster_staged_updaterun_test.go index c0490f306..99d4c0b87 100644 --- a/test/e2e/cluster_staged_updaterun_test.go +++ b/test/e2e/cluster_staged_updaterun_test.go @@ -27,6 +27,7 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -144,15 +145,17 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-1 first because of its name", func() { - checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -214,11 +217,21 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should not rollout resources to prod stage until approved", func() { + By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } + }) + + It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -301,7 +314,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -312,15 +325,17 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-1 first because of its name", func() { - checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should rollout resources to all the members after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -363,7 +378,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a new cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -381,11 +396,21 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should not rollout resources to prod stage until approved", func() { + By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } + }) + + It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -401,7 +426,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a new staged update run with old resourceSnapshotIndex successfully to rollback", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollback resources to member-cluster-2 only and completes stage canary", func() { @@ -419,11 +444,21 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{resourceSnapshotIndex2nd, resourceSnapshotIndex1st, resourceSnapshotIndex2nd}, []bool{true, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should not rollback resources to prod stage until approved", func() { + By("Verify that the configmap is not rolled back on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &newConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } + }) + + It("Should rollback resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) for idx := range allMemberClusters { configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &oldConfigMap) @@ -504,7 +539,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -515,11 +550,17 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[:2], []string{"", resourceSnapshotIndex1st}, []bool{false, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + }) + + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to member-cluster-1 too but not member-cluster-3 and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) + It("Should rollout resources to member-cluster-1 after approval but not member-cluster-3 and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -552,7 +593,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on member-cluster-1 and member-cluster-2 only and completes stage canary", func() { @@ -565,11 +606,17 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + }) + + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) }) - It("Should rollout resources to member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -601,7 +648,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -611,12 +658,14 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, false, []string{allMemberClusterNames[2]}, []string{resourceSnapshotIndex1st}, []bool{false}, nil, nil) Consistently(crpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should remove resources on member-cluster-1 and member-cluster-2 and complete the cluster staged update run successfully", func() { + It("Should remove resources on member-cluster-1 and member-cluster-2 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + // need to go through two stages - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil, true) Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -694,7 +743,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should not rollout any resources to member clusters and complete stage canary", func() { @@ -704,11 +753,17 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[2:], []string{""}, []bool{false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-3 and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) + }) + + It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[2]}) checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) @@ -741,7 +796,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on member-cluster-2 and member-cluster-3 only and completes stage canary", func() { @@ -753,11 +808,17 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{false, true, true}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + }) + + It("Should not rollout resources to member-cluster-1 until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0]}) }) - It("Should rollout resources to member-cluster-1 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should rollout resources to member-cluster-1 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[1], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -789,7 +850,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -799,11 +860,17 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames[1:], []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true}, nil, nil) Consistently(crpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary) + validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + }) + + It("Should not remove resources from member-cluster-1 until approved", func() { + checkIfPlacedWorkResourcesOnMemberClustersConsistently(allMemberClusters) }) - It("Should remove resources on member-cluster-1 and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + It("Should remove resources on member-cluster-1 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[2], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil, true) Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[1], allMemberClusters[2]}) @@ -959,7 +1026,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -971,11 +1038,17 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, wantROs) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunName, envCanary) + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs) + It("Should not rollout resources to member-cluster-1 and member-cluster-3 until approved", func() { + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + }) + + It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1062,7 +1135,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should report diff for member-cluster-2 only and completes stage canary", func() { @@ -1071,11 +1144,13 @@ var _ = Describe("test CRP rollout with staged update run", func() { []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) - validateAndApproveClusterApprovalRequests(updateRunName, envCanary) + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should report diff for member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should report diff for member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) }) @@ -1176,17 +1251,21 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Create a staged update run with new resourceSnapshotIndex and verify rollout happens", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex2nd, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateRun) // Verify rollout to canary cluster first By("Verify that the new configmap is updated on member-cluster-2 during canary stage") configMapActual := configMapPlacedOnClusterActual(allMemberClusters[1], &newConfigMap) Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s", newConfigMap.Name, allMemberClusterNames[1]) - validateAndApproveClusterApprovalRequests(updateRunName, envCanary) + // Approval for AfterStageTasks of canary stage + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + + // Approval for BeforeStageTasks of prod stage + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) // Verify complete rollout - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) // Verify new configmap is on all member clusters @@ -1246,7 +1325,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should create a staged update run and verify cluster approval request is created", func() { validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) // Verify that cluster approval request is created for canary stage. Eventually(func() error { @@ -1254,6 +1333,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ placementv1beta1.TargetUpdatingStageNameLabel: envCanary, placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, }); err != nil { return fmt.Errorf("failed to list approval requests: %w", err) } @@ -1265,54 +1345,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { }, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to find cluster approval request") }) - It("Should approve cluster approval request using kubectl-fleet approve plugin", func() { - var approvalRequestName string - - // Get the cluster approval request name. - Eventually(func() error { - appReqList := &placementv1beta1.ClusterApprovalRequestList{} - if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ - placementv1beta1.TargetUpdatingStageNameLabel: envCanary, - placementv1beta1.TargetUpdateRunLabel: updateRunName, - }); err != nil { - return fmt.Errorf("failed to list approval requests: %w", err) - } - - if len(appReqList.Items) != 1 { - return fmt.Errorf("want 1 approval request, got %d", len(appReqList.Items)) - } - - approvalRequestName = appReqList.Items[0].Name - return nil - }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to get approval request name") - - // Use kubectl-fleet approve plugin to approve the request - cmd := exec.Command(fleetBinaryPath, "approve", "clusterapprovalrequest", - "--hubClusterContext", "kind-hub", - "--name", approvalRequestName) - output, err := cmd.CombinedOutput() - Expect(err).ToNot(HaveOccurred(), "kubectl-fleet approve failed: %s", string(output)) - - // Verify the approval request is approved - Eventually(func() error { - var appReq placementv1beta1.ClusterApprovalRequest - if err := hubClient.Get(ctx, client.ObjectKey{Name: approvalRequestName}, &appReq); err != nil { - return fmt.Errorf("failed to get approval request: %w", err) - } + It("Should approve after-stage cluster approval request using kubectl-fleet approve plugin for canary stage", func() { + approveClusterApprovalRequest(envCanary, updateRunName, placementv1beta1.AfterStageTaskLabelValue) + }) - approvedCondition := meta.FindStatusCondition(appReq.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)) - if approvedCondition == nil { - return fmt.Errorf("approved condition not found") - } - if approvedCondition.Status != metav1.ConditionTrue { - return fmt.Errorf("approved condition status is %s, want True", approvedCondition.Status) - } - return nil - }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to verify approval request is approved") + It("Should approve before-stage cluster approval request using kubectl-fleet approve plugin for prod stage", func() { + approveClusterApprovalRequest(envProd, updateRunName, placementv1beta1.BeforeStageTaskLabelValue) }) It("Should complete the staged update run after approval", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1381,11 +1423,15 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Create updateRun and verify resources are rolled out", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) - validateAndApproveClusterApprovalRequests(updateRunName, envCanary) + // Approval for AfterStageTasks of canary stage + validateAndApproveClusterApprovalRequests(updateRunName, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + // Approval for BeforeStageTasks of prod stage + validateAndApproveClusterApprovalRequests(updateRunName, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -1513,13 +1559,13 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the cluster staged update run with all 3 clusters updated in parallel", func() { // With maxConcurrency=3, all 3 clusters should be updated in parallel. // Each round waits 15 seconds, so total time should be under 20s. - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunParallelEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1603,14 +1649,14 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the cluster staged update run with all 3 clusters", func() { // Since maxConcurrency=70% each round we process 2 clusters in parallel, // so all 3 clusters should be updated in 2 rounds. // Each round waits 15 seconds, so total time should be under 40s. - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunParallelEventuallyDuration*2, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1621,6 +1667,111 @@ var _ = Describe("test CRP rollout with staged update run", func() { Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) }) }) + + Context("Test resource rollout with staged update run by update run states - (Initialize -> Run)", Ordered, func() { + updateRunNames := []string{} + var strategy *placementv1beta1.ClusterStagedUpdateStrategy + + BeforeAll(func() { + // Create a test namespace and a configMap inside it on the hub cluster. + createWorkResources() + + // Create the CRP with external rollout strategy. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: workResourceSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") + + // Create the clusterStagedUpdateStrategy. + strategy = createClusterStagedUpdateStrategySucceed(strategyName) + + for i := 0; i < 1; i++ { + updateRunNames = append(updateRunNames, fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + } + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the CRP. + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + + // Remove all the clusterStagedUpdateRuns. + for _, name := range updateRunNames { + ensureClusterStagedUpdateRunDeletion(name) + } + + // Delete the clusterStagedUpdateStrategy. + ensureClusterUpdateRunStrategyDeletion(strategyName) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) + + It("Should have the latest resource snapshot", func() { + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the crp", func() { + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) + }) + + It("Should update crp status as pending rollout", func() { + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", "", ""}, []bool{false, false, false}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + }) + + It("Should create a cluster staged update run successfully", func() { + By("Creating Cluster Staged Update Run in state Initialize") + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateInitialize) + }) + + It("Should not start rollout as the update run is in Initialize state", func() { + By("Member clusters should not have work resources placed") + checkIfRemovedWorkResourcesFromAllMemberClustersConsistently() + + By("Validating the csur status remains in Initialize state") + csurNotStartedActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, false) + Consistently(csurNotStartedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to Initialize updateRun %s", updateRunNames[0]) + }) + + It("Should rollout resources to member-cluster-2 only after update run is in Run state", func() { + // Update the update run state to Run + By("Updating the update run state to Run") + updateClusterStagedUpdateRunState(updateRunNames[0], placementv1beta1.StateRun) + + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating crp status as member-cluster-2 updated") + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + }) + + It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { + validateAndApproveClusterApprovalRequests(updateRunNames[0], envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) + }) + + It("Should update crp status as completed", func() { + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + }) + }) }) // Note that this container cannot run in parallel with other containers. @@ -1688,10 +1839,10 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) By("Creating the first staged update run") - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) By("Validating staged update run has succeeded") - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) By("Validating CRP status as completed") @@ -1739,11 +1890,11 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should create another staged update run for the same CRP", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 2) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the second staged update run and complete the CRP", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1], allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1], allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames[1:], @@ -1787,11 +1938,11 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the staged update run, complete CRP, and rollout resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, @@ -1830,11 +1981,11 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateRun) }) It("Should complete the staged update run, complete CRP, and rollout updated resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex2nd, true, allMemberClusterNames, @@ -1869,11 +2020,11 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the staged update run, complete CRP, and re-place resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, @@ -1956,6 +2107,11 @@ func createClusterStagedUpdateStrategySucceed(strategyName string) *placementv1b envLabelName: envProd, // member-cluster-1 and member-cluster-3 }, }, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, }, }, }, @@ -2010,12 +2166,13 @@ func validateLatestClusterResourceSnapshot(crpName, wantResourceSnapshotIndex st }, eventuallyDuration, eventuallyInterval).Should(Equal(wantResourceSnapshotIndex), "Resource snapshot index does not match") } -func createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex, strategyName string) { +func createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex, strategyName string, state placementv1beta1.State) { updateRun := &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ + State: state, PlacementName: crpName, ResourceSnapshotIndex: resourceSnapshotIndex, StagedUpdateStrategyName: strategyName, @@ -2030,6 +2187,7 @@ func createClusterStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNam Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ + State: placementv1beta1.StateRun, PlacementName: crpName, StagedUpdateStrategyName: strategyName, }, @@ -2037,12 +2195,28 @@ func createClusterStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNam Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create ClusterStagedUpdateRun %s", updateRunName) } -func validateAndApproveClusterApprovalRequests(updateRunName, stageName string) { +func updateClusterStagedUpdateRunState(updateRunName string, state placementv1beta1.State) { + Eventually(func() error { + updateRun := &placementv1beta1.ClusterStagedUpdateRun{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName}, updateRun); err != nil { + return fmt.Errorf("failed to get ClusterStagedUpdateRun %s", updateRunName) + } + + updateRun.Spec.State = state + if err := hubClient.Update(ctx, updateRun); err != nil { + return fmt.Errorf("failed to update ClusterStagedUpdateRun %s", updateRunName) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update ClusterStagedUpdateRun %s state to %s", updateRunName, state) +} + +func validateAndApproveClusterApprovalRequests(updateRunName, stageName, approvalRequestNameFmt, stageTaskType string) { Eventually(func() error { appReqList := &placementv1beta1.ClusterApprovalRequestList{} if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: stageTaskType, }); err != nil { return fmt.Errorf("failed to list approval requests: %w", err) } @@ -2051,6 +2225,10 @@ func validateAndApproveClusterApprovalRequests(updateRunName, stageName string) return fmt.Errorf("got %d approval requests, want 1", len(appReqList.Items)) } appReq := &appReqList.Items[0] + approvalRequestName := fmt.Sprintf(approvalRequestNameFmt, updateRunName, stageName) + if appReq.Name != approvalRequestName { + return fmt.Errorf("got approval request %s, want %s", appReq.Name, approvalRequestName) + } meta.SetStatusCondition(&appReq.Status.Conditions, metav1.Condition{ Status: metav1.ConditionTrue, Type: string(placementv1beta1.ApprovalRequestConditionApproved), @@ -2068,3 +2246,50 @@ func updateConfigMapSucceed(newConfigMap *corev1.ConfigMap) { cm.Data = newConfigMap.Data Expect(hubClient.Update(ctx, cm)).To(Succeed(), "Failed to update configmap %s in namespace %s", newConfigMap.Name, newConfigMap.Namespace) } + +func approveClusterApprovalRequest(stageName, updateRunName, stageTask string) { + var approvalRequestName string + + // Get the cluster approval request name. + Eventually(func() error { + appReqList := &placementv1beta1.ClusterApprovalRequestList{} + if err := hubClient.List(ctx, appReqList, client.MatchingLabels{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: stageTask, + }); err != nil { + return fmt.Errorf("failed to list approval requests: %w", err) + } + + if len(appReqList.Items) != 1 { + return fmt.Errorf("want 1 approval request, got %d", len(appReqList.Items)) + } + + approvalRequestName = appReqList.Items[0].Name + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to get approval request name") + + // Use kubectl-fleet approve plugin to approve the request + cmd := exec.Command(fleetBinaryPath, "approve", "clusterapprovalrequest", + "--hubClusterContext", "kind-hub", + "--name", approvalRequestName) + output, err := cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred(), "kubectl-fleet approve failed: %s", string(output)) + + // Verify the approval request is approved + Eventually(func() error { + var appReq placementv1beta1.ClusterApprovalRequest + if err := hubClient.Get(ctx, client.ObjectKey{Name: approvalRequestName}, &appReq); err != nil { + return fmt.Errorf("failed to get approval request: %w", err) + } + + approvedCondition := meta.FindStatusCondition(appReq.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)) + if approvedCondition == nil { + return fmt.Errorf("approved condition not found") + } + if approvedCondition.Status != metav1.ConditionTrue { + return fmt.Errorf("approved condition status is %s, want True", approvedCondition.Status) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to verify approval request is approved") +} diff --git a/test/e2e/enveloped_object_placement_test.go b/test/e2e/enveloped_object_placement_test.go index bf7c707b5..5ddc19a93 100644 --- a/test/e2e/enveloped_object_placement_test.go +++ b/test/e2e/enveloped_object_placement_test.go @@ -226,7 +226,7 @@ var _ = Describe("placing wrapped resources using a CRP", func() { // read the test resources. readDeploymentTestManifest(&testDeployment) readDaemonSetTestManifest(&testDaemonSet) - readStatefulSetTestManifest(&testStatefulSet, true) + readStatefulSetTestManifest(&testStatefulSet, StatefulSetInvalidStorage) readEnvelopeResourceTestManifest(&testResourceEnvelope) }) diff --git a/test/e2e/join_and_leave_test.go b/test/e2e/join_and_leave_test.go index 4bafee8d1..5286cc052 100644 --- a/test/e2e/join_and_leave_test.go +++ b/test/e2e/join_and_leave_test.go @@ -17,9 +17,7 @@ limitations under the License. package e2e import ( - "errors" "fmt" - "reflect" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -163,34 +161,8 @@ var _ = Describe("Test member cluster join and leave flow", Label("joinleave"), } }) - It("Should fail the unjoin requests", func() { - for idx := range allMemberClusters { - memberCluster := allMemberClusters[idx] - mcObj := &clusterv1beta1.MemberCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: memberCluster.ClusterName, - }, - } - err := hubClient.Delete(ctx, mcObj) - Expect(err).ShouldNot(Succeed(), "Want the deletion to be denied") - var statusErr *apierrors.StatusError - Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Delete memberCluster call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&apierrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("Please delete serviceExport test-namespace/test-svc in the member cluster before leaving, request is denied")) - } - }) - - It("Deleting the internalServiceExports", func() { - for idx := range allMemberClusterNames { - memberCluster := allMemberClusters[idx] - namespaceName := fmt.Sprintf(utils.NamespaceNameFormat, memberCluster.ClusterName) - - internalSvcExportKey := types.NamespacedName{Namespace: namespaceName, Name: internalServiceExportName} - var export fleetnetworkingv1alpha1.InternalServiceExport - Expect(hubClient.Get(ctx, internalSvcExportKey, &export)).Should(Succeed(), "Failed to get internalServiceExport") - Expect(hubClient.Delete(ctx, &export)).To(Succeed(), "Failed to delete internalServiceExport") - } - }) - + // The network agent is not turned on by default in the e2e so we are still able to leave when we have internalServiceExport + // TODO: add a test case for the network agent is turned on in the fleet network repository. It("Should be able to trigger the member cluster DELETE", func() { setAllMemberClustersToLeave() }) @@ -362,22 +334,7 @@ var _ = Describe("Test member cluster join and leave flow", Label("joinleave"), } }) - It("Should fail the unjoin requests", func() { - for idx := range allMemberClusters { - memberCluster := allMemberClusters[idx] - mcObj := &clusterv1beta1.MemberCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: memberCluster.ClusterName, - }, - } - err := hubClient.Delete(ctx, mcObj) - Expect(err).ShouldNot(Succeed(), "Want the deletion to be denied") - var statusErr *apierrors.StatusError - Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Delete memberCluster call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&apierrors.StatusError{}))) - Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("Please delete serviceExport test-namespace/test-svc in the member cluster before leaving, request is denied")) - } - }) - + // It does not really matter here as the network agent is not turned on by default in the e2e so we are still able to leave when we have internalServiceExport It("Updating the member cluster to skip validation", func() { for idx := range allMemberClusterNames { memberCluster := allMemberClusters[idx] diff --git a/test/e2e/placement_selecting_resources_test.go b/test/e2e/placement_selecting_resources_test.go index 3151ca384..10d6a56c3 100644 --- a/test/e2e/placement_selecting_resources_test.go +++ b/test/e2e/placement_selecting_resources_test.go @@ -1392,11 +1392,11 @@ var _ = Describe("creating CRP and checking selected resources order", Ordered, It("should update CRP status with the correct order of the selected resources", func() { // Define the expected resources in order + // Note: PVCs are not propagated, so they should not appear in selected resources expectedResources := []placementv1beta1.ResourceIdentifier{ {Kind: "Namespace", Name: nsName, Version: "v1"}, {Kind: "Secret", Name: secret.Name, Namespace: nsName, Version: "v1"}, {Kind: "ConfigMap", Name: configMap.Name, Namespace: nsName, Version: "v1"}, - {Kind: "PersistentVolumeClaim", Name: pvc.Name, Namespace: nsName, Version: "v1"}, {Group: "rbac.authorization.k8s.io", Kind: "Role", Name: role.Name, Namespace: nsName, Version: "v1"}, } diff --git a/test/e2e/resource_placement_hub_workload_test.go b/test/e2e/resource_placement_hub_workload_test.go index 852dcd3c5..b088109ee 100644 --- a/test/e2e/resource_placement_hub_workload_test.go +++ b/test/e2e/resource_placement_hub_workload_test.go @@ -19,6 +19,7 @@ package e2e import ( "fmt" + "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" @@ -37,12 +38,14 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res var testDeployment appsv1.Deployment var testDaemonSet appsv1.DaemonSet var testJob batchv1.Job + var testStatefulSet appsv1.StatefulSet BeforeAll(func() { // Read the test manifests readDeploymentTestManifest(&testDeployment) readDaemonSetTestManifest(&testDaemonSet) readJobTestManifest(&testJob) + readStatefulSetTestManifest(&testStatefulSet, StatefulSetWithStorage) workNamespace := appNamespace() // Create namespace and workloads @@ -51,9 +54,11 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res testDeployment.Namespace = workNamespace.Name testDaemonSet.Namespace = workNamespace.Name testJob.Namespace = workNamespace.Name + testStatefulSet.Namespace = workNamespace.Name Expect(hubClient.Create(ctx, &testDeployment)).To(Succeed(), "Failed to create test deployment %s", testDeployment.Name) Expect(hubClient.Create(ctx, &testDaemonSet)).To(Succeed(), "Failed to create test daemonset %s", testDaemonSet.Name) Expect(hubClient.Create(ctx, &testJob)).To(Succeed(), "Failed to create test job %s", testJob.Name) + Expect(hubClient.Create(ctx, &testStatefulSet)).To(Succeed(), "Failed to create test statefulset %s", testStatefulSet.Name) // Create the CRP that selects the namespace By("creating CRP that selects the namespace") @@ -105,9 +110,16 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res Name: testJob.Name, Namespace: workNamespace.Name, }, + { + Group: "apps", + Version: "v1", + Kind: "StatefulSet", + Name: testStatefulSet.Name, + Namespace: workNamespace.Name, + }, } // Use customizedPlacementStatusUpdatedActual with resourceIsTrackable=false - // because Jobs don't have availability tracking like Deployments/DaemonSets do + // because Jobs don't have availability tracking like Deployments/DaemonSets/StatefulSets do crpKey := types.NamespacedName{Name: crpName} crpStatusUpdatedActual := customizedPlacementStatusUpdatedActual(crpKey, wantSelectedResources, allMemberClusterNames, nil, "0", false) Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") @@ -170,6 +182,13 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res "Hub job should complete successfully") }) + It("should verify hub statefulset is ready", func() { + By("checking hub statefulset status") + statefulSetReadyActual := waitForStatefulSetToBeReady(hubClient, &testStatefulSet) + Eventually(statefulSetReadyActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), + "Hub statefulset should be ready before placement") + }) + It("should place the deployment on all member clusters", func() { By("verifying deployment is placed and ready on all member clusters") for idx := range allMemberClusters { @@ -206,6 +225,24 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res } }) + It("should place the statefulset on all member clusters", func() { + By("verifying statefulset is placed and ready on all member clusters") + for idx := range allMemberClusters { + memberCluster := allMemberClusters[idx] + statefulsetPlacedActual := waitForStatefulSetPlacementToReady(memberCluster, &testStatefulSet) + Eventually(statefulsetPlacedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place statefulset on member cluster %s", memberCluster.ClusterName) + } + }) + + It("should verify statefulset replicas are ready on all clusters", func() { + By("checking statefulset status on each cluster") + for _, cluster := range allMemberClusters { + statefulSetReadyActual := waitForStatefulSetToBeReady(cluster.KubeClient, &testStatefulSet) + Eventually(statefulSetReadyActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), + "StatefulSet should be ready on cluster %s", cluster.ClusterName) + } + }) + It("should verify deployment replicas are ready on all clusters", func() { By("checking deployment status on each cluster") for _, cluster := range allMemberClusters { @@ -232,6 +269,42 @@ var _ = Describe("placing workloads using a CRP with PickAll policy", Label("res }) }) +func waitForStatefulSetToBeReady(kubeClient client.Client, testStatefulSet *appsv1.StatefulSet) func() error { + return func() error { + var statefulSet appsv1.StatefulSet + if err := kubeClient.Get(ctx, types.NamespacedName{ + Name: testStatefulSet.Name, + Namespace: testStatefulSet.Namespace, + }, &statefulSet); err != nil { + return err + } + + // Verify statefulset is ready + requiredReplicas := int32(1) + if statefulSet.Spec.Replicas != nil { + requiredReplicas = *statefulSet.Spec.Replicas + } + + wantStatus := appsv1.StatefulSetStatus{ + ObservedGeneration: statefulSet.Generation, + CurrentReplicas: requiredReplicas, + UpdatedReplicas: requiredReplicas, + } + + gotStatus := appsv1.StatefulSetStatus{ + ObservedGeneration: statefulSet.Status.ObservedGeneration, + CurrentReplicas: statefulSet.Status.CurrentReplicas, + UpdatedReplicas: statefulSet.Status.UpdatedReplicas, + } + + if diff := cmp.Diff(wantStatus, gotStatus); diff != "" { + return fmt.Errorf("statefulset not ready (-want +got):\n%s", diff) + } + + return nil + } +} + func waitForJobToComplete(kubeClient client.Client, testJob *batchv1.Job) func() error { return func() error { var job batchv1.Job diff --git a/test/e2e/resource_placement_rollout_test.go b/test/e2e/resource_placement_rollout_test.go index b537b36d3..b07491111 100644 --- a/test/e2e/resource_placement_rollout_test.go +++ b/test/e2e/resource_placement_rollout_test.go @@ -69,7 +69,7 @@ var _ = Describe("placing namespaced scoped resources using a RP with rollout", testDaemonSet = appv1.DaemonSet{} readDaemonSetTestManifest(&testDaemonSet) testStatefulSet = appv1.StatefulSet{} - readStatefulSetTestManifest(&testStatefulSet, false) + readStatefulSetTestManifest(&testStatefulSet, StatefulSetBasic) testService = corev1.Service{} readServiceTestManifest(&testService) testJob = batchv1.Job{} diff --git a/test/e2e/resource_placement_selecting_resources_test.go b/test/e2e/resource_placement_selecting_resources_test.go index 4817784ff..31beca108 100644 --- a/test/e2e/resource_placement_selecting_resources_test.go +++ b/test/e2e/resource_placement_selecting_resources_test.go @@ -1013,10 +1013,10 @@ var _ = Describe("testing RP selecting resources", Label("resourceplacement"), f It("should update RP status with the correct order of the selected resources", func() { // Define the expected resources in order. + // Note: PVCs are not propagated, so they should not appear in selected resources expectedResources := []placementv1beta1.ResourceIdentifier{ {Kind: "Secret", Name: secret.Name, Namespace: nsName, Version: "v1"}, {Kind: "ConfigMap", Name: configMap.Name, Namespace: nsName, Version: "v1"}, - {Kind: "PersistentVolumeClaim", Name: pvc.Name, Namespace: nsName, Version: "v1"}, {Group: "rbac.authorization.k8s.io", Kind: "Role", Name: role.Name, Namespace: nsName, Version: "v1"}, } diff --git a/test/e2e/resources/test-statefulset.yaml b/test/e2e/resources/statefulset-basic.yaml similarity index 100% rename from test/e2e/resources/test-statefulset.yaml rename to test/e2e/resources/statefulset-basic.yaml diff --git a/test/e2e/resources/statefulset-with-volume.yaml b/test/e2e/resources/statefulset-invalid-storage.yaml similarity index 100% rename from test/e2e/resources/statefulset-with-volume.yaml rename to test/e2e/resources/statefulset-invalid-storage.yaml diff --git a/test/e2e/resources/statefulset-with-storage.yaml b/test/e2e/resources/statefulset-with-storage.yaml new file mode 100644 index 000000000..a9df178e5 --- /dev/null +++ b/test/e2e/resources/statefulset-with-storage.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-ss +spec: + selector: + matchLabels: + app: test-ss + serviceName: "test-ss-svc" + replicas: 2 + template: + metadata: + labels: + app: test-ss + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: pause + image: k8s.gcr.io/pause:3.8 + volumeClaimTemplates: + - metadata: + name: test-ss-pvc + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "standard" + resources: + requests: + storage: 100Mi diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index 6c0157271..d52acb287 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -342,7 +342,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { BeforeAll(func() { // Create the test resources. - readStatefulSetTestManifest(&testStatefulSet, false) + readStatefulSetTestManifest(&testStatefulSet, StatefulSetBasic) readEnvelopeResourceTestManifest(&testStatefulSetEnvelope) wantSelectedResources = []placementv1beta1.ResourceIdentifier{ { diff --git a/test/e2e/staged_updaterun_test.go b/test/e2e/staged_updaterun_test.go index 76f7e1d14..94ebc5684 100644 --- a/test/e2e/staged_updaterun_test.go +++ b/test/e2e/staged_updaterun_test.go @@ -135,15 +135,17 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-1 first because of its name", func() { - checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to all the members and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -204,11 +206,20 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + }) + It("Should not rollout resources to prod stage until approved", func() { + By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -289,7 +300,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -300,15 +311,17 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-1 first because of its name", func() { - checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) }) - It("Should rollout resources to all the members and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should rollout resources to all the members after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -351,7 +364,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a new staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex2nd, strategyName) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -369,11 +382,21 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + }) + + It("Should not rollout resources to prod stage until approved", func() { + By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -389,7 +412,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a new staged update run with old resourceSnapshotIndex successfully to rollback", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollback resources to member-cluster-2 only and completes stage canary", func() { @@ -407,11 +430,21 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{resourceSnapshotIndex2nd, resourceSnapshotIndex1st, resourceSnapshotIndex2nd}, []bool{true, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + }) + + It("Should not rollback resources to prod stage until approved", func() { + By("Verify that the configmap is not rolled back on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &newConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } }) - It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should rollback resources to member-cluster-1 and member-cluster-3 after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) for idx := range allMemberClusters { configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &oldConfigMap) @@ -490,7 +523,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -501,11 +534,17 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[:2], []string{"", resourceSnapshotIndex1st}, []bool{false, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-1 too but not member-cluster-3 and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + }) + + It("Should rollout resources to member-cluster-1 after approval but not member-cluster-3 and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -538,7 +577,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on member-cluster-1 and member-cluster-2 only and completes stage canary", func() { @@ -551,11 +590,17 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) + }) + + It("Should rollout resources to member-cluster-3 after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -587,7 +632,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -597,12 +642,14 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, false, []string{allMemberClusterNames[2]}, []string{resourceSnapshotIndex1st}, []bool{false}, nil, nil) Consistently(rpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should remove resources on member-cluster-1 and member-cluster-2 and complete the staged update run successfully", func() { + It("Should remove resources on member-cluster-1 and member-cluster-2 after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + // need to go through two stages - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil, true) Eventually(surSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[2]) checkIfRemovedConfigMapFromMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -678,7 +725,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should not rollout any resources to member clusters and complete stage canary", func() { @@ -688,11 +735,17 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames[2:], []string{""}, []bool{false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-3 and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) + It("Should not rollout resources to prod stage until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) + }) + + It("Should rollout resources to member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[2]}) checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) @@ -725,7 +778,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on member-cluster-2 and member-cluster-3 only and completes stage canary", func() { @@ -737,11 +790,17 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{false, true, true}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to keep RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + }) + + It("Should not rollout resources to member-cluster-1 until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0]}) }) - It("Should rollout resources to member-cluster-1 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should rollout resources to member-cluster-1 after approval and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -773,7 +832,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -783,11 +842,17 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames[1:], []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true}, nil, nil) Consistently(rpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + }) + + It("Should not remove resources from member-cluster-1 until approved", func() { + checkIfPlacedWorkResourcesOnMemberClustersConsistently(allMemberClusters) }) - It("Should remove resources on member-cluster-1 and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + It("Should remove resources on member-cluster-1 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil, true) Eventually(surSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[2]) checkIfRemovedConfigMapFromMemberClusters([]*framework.Cluster{allMemberClusters[0]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[1], allMemberClusters[2]}) @@ -915,7 +980,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -928,11 +993,17 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, wantROs) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, wantROs) + It("Should not rollout resources to member-cluster-1 and member-cluster-3 until approved", func() { + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + }) + + It("Should rollout resources to member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, wantROs, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1013,7 +1084,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should report diff for member-cluster-2 only and completes stage canary", func() { @@ -1022,11 +1093,13 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) }) - It("Should report diff for member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + It("Should report diff for member-cluster-1 and member-cluster-3 after approval and complete the cluster staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) }) @@ -1125,17 +1198,21 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Create a staged update run with new resourceSnapshotIndex and verify rollout happens", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex2nd, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateRun) // Verify rollout to canary cluster first. By("Verify that the new configmap is updated on member-cluster-2 during canary stage") configMapActual := configMapPlacedOnClusterActual(allMemberClusters[1], &newConfigMap) Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s", newConfigMap.Name, allMemberClusterNames[1]) - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) + // Approval for AfterStageTask of canary stage + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + + // Approval for BeforeStageTask of prod stage + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) // Verify complete rollout. - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) // Verify new configmap is on all member clusters. @@ -1207,11 +1284,15 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Create updateRun and verify resources are rolled out", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) - validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) + // Approval for AfterStageTask of canary stage + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + // Approval for BeforeStageTask of prod stage + validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -1338,13 +1419,13 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the staged update run with all 3 clusters updated in parallel", func() { // With maxConcurrency=3, all 3 clusters should be updated in parallel. // Each round waits 15 seconds, so total time should be under 20s. - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunParallelEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1427,14 +1508,14 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateRun) }) It("Should complete the staged update run with all 3 clusters", func() { // Since maxConcurrency=70% each round we process 2 clusters in parallel, // so all 3 clusters should be updated in 2 rounds. // Each round waits 15 seconds, so total time should be under 40s. - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil, true) Eventually(surSucceededActual, updateRunParallelEventuallyDuration*2, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1445,6 +1526,109 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) }) }) + + Context("Test resource rollout with staged update run by update run states - (Initialize -> Run)", Ordered, func() { + updateRunNames := []string{} + var strategy *placementv1beta1.StagedUpdateStrategy + + BeforeAll(func() { + // Create the RP with external rollout strategy. + rp := &placementv1beta1.ResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: rpName, + Namespace: testNamespace, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: configMapSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, rp)).To(Succeed(), "Failed to create RP") + + // Create the stagedUpdateStrategy. + strategy = createStagedUpdateStrategySucceed(strategyName, testNamespace) + + for i := 0; i < 3; i++ { + updateRunNames = append(updateRunNames, fmt.Sprintf(stagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + } + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the RP. + ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: testNamespace}, allMemberClusters) + + // Remove all the stagedUpdateRuns. + for _, name := range updateRunNames { + ensureStagedUpdateRunDeletion(name, testNamespace) + } + + // Delete the stagedUpdateStrategy. + ensureStagedUpdateRunStrategyDeletion(strategyName, testNamespace) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedConfigMapFromAllMemberClustersConsistently) + + It("Should have the latest resource snapshot", func() { + validateLatestResourceSnapshot(rpName, testNamespace, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the rp", func() { + validateLatestSchedulingPolicySnapshot(rpName, testNamespace, policySnapshotIndex1st, 3) + }) + + It("Should update rp status as pending rollout", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", "", ""}, []bool{false, false, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + }) + + It("Should create a staged update run successfully", func() { + By("Creating staged update run in Initialize state") + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateInitialize) + }) + + It("Should not start rollout as the update run is in Initialize state", func() { + By("Member clusters should not have work resources placed") + checkIfRemovedConfigMapFromAllMemberClustersConsistently() + + By("Validating the sur status remains in Initialize state") + surNotStartedActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, false) + Consistently(surNotStartedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to Initialize updateRun %s/%s ", testNamespace, updateRunNames[0]) + }) + + It("Should rollout resources to member-cluster-2 only after update run is in Run state", func() { + // Update the update run state to Run. + By("Updating the update run state to Run") + updateStagedUpdateRunState(updateRunNames[0], testNamespace, placementv1beta1.StateRun) + + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating crp status as member-cluster-2 updated") + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary, placementv1beta1.AfterStageApprovalTaskNameFmt, placementv1beta1.AfterStageTaskLabelValue) + }) + + It("Should rollout resources to all the members and complete the staged update run successfully", func() { + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envProd, placementv1beta1.BeforeStageApprovalTaskNameFmt, placementv1beta1.BeforeStageTaskLabelValue) + + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil, true) + Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) + }) + + It("Should update rp status as completed", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + }) + }) }) func createStagedUpdateStrategySucceed(strategyName, namespace string) *placementv1beta1.StagedUpdateStrategy { @@ -1481,6 +1665,11 @@ func createStagedUpdateStrategySucceed(strategyName, namespace string) *placemen envLabelName: envProd, // member-cluster-1 and member-cluster-3 }, }, + BeforeStageTasks: []placementv1beta1.StageTask{ + { + Type: placementv1beta1.StageTaskTypeApproval, + }, + }, }, }, }, @@ -1535,7 +1724,7 @@ func validateLatestResourceSnapshot(rpName, namespace, wantResourceSnapshotIndex }, eventuallyDuration, eventuallyInterval).Should(Equal(wantResourceSnapshotIndex), "Resource snapshot index does not match") } -func createStagedUpdateRunSucceed(updateRunName, namespace, rpName, resourceSnapshotIndex, strategyName string) { +func createStagedUpdateRunSucceed(updateRunName, namespace, rpName, resourceSnapshotIndex, strategyName string, state placementv1beta1.State) { updateRun := &placementv1beta1.StagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, @@ -1543,6 +1732,7 @@ func createStagedUpdateRunSucceed(updateRunName, namespace, rpName, resourceSnap }, Spec: placementv1beta1.UpdateRunSpec{ PlacementName: rpName, + State: state, ResourceSnapshotIndex: resourceSnapshotIndex, StagedUpdateStrategyName: strategyName, }, @@ -1557,6 +1747,7 @@ func createStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunName, name Namespace: namespace, }, Spec: placementv1beta1.UpdateRunSpec{ + State: placementv1beta1.StateRun, PlacementName: rpName, StagedUpdateStrategyName: strategyName, }, @@ -1564,12 +1755,28 @@ func createStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunName, name Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create StagedUpdateRun %s", updateRunName) } -func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stageName string) { +func updateStagedUpdateRunState(updateRunName, namespace string, state placementv1beta1.State) { + Eventually(func() error { + updateRun := &placementv1beta1.StagedUpdateRun{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName, Namespace: namespace}, updateRun); err != nil { + return fmt.Errorf("failed to get StagedUpdateRun %s", updateRunName) + } + + updateRun.Spec.State = state + if err := hubClient.Update(ctx, updateRun); err != nil { + return fmt.Errorf("failed to update StagedUpdateRun %s", updateRunName) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update StagedUpdateRun %s to state %s", updateRunName, state) +} + +func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stageName, approvalRequestNameFmt, stageTaskType string) { Eventually(func() error { appReqList := &placementv1beta1.ApprovalRequestList{} if err := hubClient.List(ctx, appReqList, client.InNamespace(namespace), client.MatchingLabels{ placementv1beta1.TargetUpdatingStageNameLabel: stageName, placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.TaskTypeLabel: stageTaskType, }); err != nil { return fmt.Errorf("failed to list approval requests: %w", err) } @@ -1578,6 +1785,10 @@ func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stag return fmt.Errorf("got %d approval requests, want 1", len(appReqList.Items)) } appReq := &appReqList.Items[0] + approvalRequestName := fmt.Sprintf(approvalRequestNameFmt, updateRunName, stageName) + if appReq.Name != approvalRequestName { + return fmt.Errorf("got approval request %s, want %s", appReq.Name, approvalRequestName) + } meta.SetStatusCondition(&appReq.Status.Conditions, metav1.Condition{ Status: metav1.ConditionTrue, Type: string(placementv1beta1.ApprovalRequestConditionApproved), diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 4f2cca56b..15318d9b9 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -57,6 +57,18 @@ import ( "go.goms.io/fleet/test/e2e/framework" ) +// StatefulSetVariant represents different StatefulSet configurations for testing +type StatefulSetVariant int + +const ( + // StatefulSetBasic is a StatefulSet without any persistent volume claims + StatefulSetBasic StatefulSetVariant = iota + // StatefulSetInvalidStorage is a StatefulSet with a non-existent storage class + StatefulSetInvalidStorage + // StatefulSetWithStorage is a StatefulSet with a valid standard storage class + StatefulSetWithStorage +) + var ( croTestAnnotationKey = "cro-test-annotation" croTestAnnotationValue = "cro-test-annotation-val" @@ -1537,13 +1549,18 @@ func readDaemonSetTestManifest(testDaemonSet *appsv1.DaemonSet) { Expect(err).Should(Succeed()) } -func readStatefulSetTestManifest(testStatefulSet *appsv1.StatefulSet, withVolume bool) { +func readStatefulSetTestManifest(testStatefulSet *appsv1.StatefulSet, variant StatefulSetVariant) { By("Read the statefulSet resource") - if withVolume { - Expect(utils.GetObjectFromManifest("resources/statefulset-with-volume.yaml", testStatefulSet)).Should(Succeed()) - } else { - Expect(utils.GetObjectFromManifest("resources/test-statefulset.yaml", testStatefulSet)).Should(Succeed()) - } + var manifestPath string + switch variant { + case StatefulSetBasic: + manifestPath = "resources/statefulset-basic.yaml" + case StatefulSetInvalidStorage: + manifestPath = "resources/statefulset-invalid-storage.yaml" + case StatefulSetWithStorage: + manifestPath = "resources/statefulset-with-storage.yaml" + } + Expect(utils.GetObjectFromManifest(manifestPath, testStatefulSet)).Should(Succeed()) } func readServiceTestManifest(testService *corev1.Service) {