Skip to content
This repository was archived by the owner on May 15, 2023. It is now read-only.
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 5 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -174,19 +174,13 @@ The following command will show which compute worker node got disabled to be rem

#### Remove a specific compute worker node

Edit the computenodeopenstack CR, lower the workers number and add a `nodesToDelete` section in the spec with the details of the worker node which should be removed:
Annotate the machine object corresponding to the node to be removed:

oc -n openstack edit computenodeopenstacks.compute-node.openstack.org example-computenodeopenstack

Modify the number of `workers` and add the `nodesToDelete` section to the spec:
oc get -n openshift-machine-api machine -o wide
oc annotate -n openshift-machine-api machines <machine-name> machine.openshift.io/cluster-api-delete-machine=1

nodesToDelete:
- name: <name of the compute worker node, e.g. worker-3>
# enable disable live migration for this node. If not specified the global setting from the
# `drain:` section is used. If not specified there, the default is `false`
drain: true/false

Save and exit.
Edit the computenodeopenstack CR, lower the workers number, save and exit:
oc -n openstack edit computenodeopenstacks.compute-node.openstack.org example-computenodeopenstack

## Cleanup

Expand Down
12 changes: 0 additions & 12 deletions api/v1alpha1/computenodeopenstack_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@ type ComputeNodeOpenStackSpec struct {
Dedicated bool `json:"dedicated,omitempty"`
// Infra DaemonSets needed
InfraDaemonSets []InfraDaemonSet `json:"infraDaemonSets,omitempty"`
// Nodes to delete upon scale down
NodesToDelete []NodeToDelete `json:"nodesToDelete,omitempty"`
// openstackclient configmap which holds information to connect to OpenStack API
OpenStackClientConfigMap string `json:"openStackClientConfigMap"`
// user secrets used to connect to OpenStack API via openstackclient
Expand Down Expand Up @@ -99,14 +97,6 @@ type SriovConfig struct {
DevName string `json:"devName"`
}

// NodeToDelete defines the name of the node to delete and if automatic drain is needed
type NodeToDelete struct {
// Node Name
Name string `json:"name"`
// Automatic draining of the node
Drain bool `json:"drain,omitempty"`
}

// Node defines the status of the associated nodes
type Node struct {
// Node name
Expand All @@ -129,8 +119,6 @@ type ComputeNodeOpenStackStatus struct {
SpecMDS string `json:"specMDS"`
// Nodes information
Nodes []Node `json:"nodes,omitempty"`
// Nodes to delete upon scale down
NodesToDelete []NodeToDelete `json:"nodesToDelete,omitempty"`
}

// +kubebuilder:object:root=true
Expand Down
25 changes: 0 additions & 25 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -125,22 +125,6 @@ spec:
required:
- nic
type: object
nodesToDelete:
description: Nodes to delete upon scale down
items:
description: NodeToDelete defines the name of the node to delete and
if automatic drain is needed
properties:
drain:
description: Automatic draining of the node
type: boolean
name:
description: Node Name
type: string
required:
- name
type: object
type: array
openStackClientAdminSecret:
description: user secrets used to connect to OpenStack API via openstackclient
type: string
Expand Down Expand Up @@ -205,22 +189,6 @@ spec:
- status
type: object
type: array
nodesToDelete:
description: Nodes to delete upon scale down
items:
description: NodeToDelete defines the name of the node to delete and
if automatic drain is needed
properties:
drain:
description: Automatic draining of the node
type: boolean
name:
description: Node Name
type: string
required:
- name
type: object
type: array
readyWorkers:
description: Number of ready workers
format: int32
Expand Down
90 changes: 2 additions & 88 deletions controllers/computenodeopenstack_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,12 +190,6 @@ func (r *ComputeNodeOpenStackReconciler) Reconcile(req ctrl.Request) (ctrl.Resul
if err != nil {
return ctrl.Result{}, err
}
} else if !reflect.DeepEqual(instance.Spec.NodesToDelete, instance.Status.NodesToDelete) || instance.Spec.Workers < instance.Status.Workers {
// Check if nodes to delete information has changed
err := updateMachineDeletionSelection(r.Client, instance)
if err != nil {
return ctrl.Result{}, err
}
}

// Need to reapply the spec
Expand Down Expand Up @@ -273,7 +267,6 @@ func (r *ComputeNodeOpenStackReconciler) Reconcile(req ctrl.Request) (ctrl.Resul
// Update Status
instance.Status.Workers = instance.Spec.Workers
instance.Status.InfraDaemonSets = instance.Spec.InfraDaemonSets
instance.Status.NodesToDelete = instance.Spec.NodesToDelete
instance.Status.SpecMDS = specMDS
err = r.Client.Status().Update(context.TODO(), instance)
if err != nil {
Expand Down Expand Up @@ -571,7 +564,7 @@ func (r *ComputeNodeOpenStackReconciler) updateNodesStatus(instance *computenode
/* Steps to delete the drain the node
1. Predraining: disable nova service, (optional) migrate VMs, wait until there is no VMs
2. Postdraining: taint the node, remove nova-compute from nova services and placement
3. Remove cleanup jobs, blocker pod finalizer, and update nodesToDelete status information
3. Remove cleanup jobs, blocker pod finalizer
*/
// 1. NodePreDrain
nodePreDrained, err := ensureNodePreDrain(r.Client, node.Name, instance, openstackClientAdmin, openstackClient)
Expand Down Expand Up @@ -629,22 +622,6 @@ func (r *ComputeNodeOpenStackReconciler) updateNodesStatus(instance *computenode
if err != nil {
return err
}

log.Info(fmt.Sprintf("Updating nodeToDelete status"))
for i, nodeToDelete := range instance.Spec.NodesToDelete {
if nodeToDelete.Name == node.Name {
if len(instance.Spec.NodesToDelete) == 1 {
instance.Spec.NodesToDelete = []computenodev1alpha1.NodeToDelete{}
} else {
instance.Spec.NodesToDelete = removeNode(instance.Spec.NodesToDelete, i)
}
err = r.Client.Update(context.TODO(), instance)
if err != nil {
return err
}
break
}
}
}
}

Expand All @@ -659,11 +636,6 @@ func (r *ComputeNodeOpenStackReconciler) updateNodesStatus(instance *computenode
return nil
}

func removeNode(nodes []computenodev1alpha1.NodeToDelete, i int) []computenodev1alpha1.NodeToDelete {
nodes[i] = nodes[len(nodes)-1]
return nodes[:len(nodes)-1]
}

func getNodeStatus(c client.Client, node *corev1.Node) string {
if node.Spec.Unschedulable {
return "SchedulingDisabled"
Expand Down Expand Up @@ -851,15 +823,6 @@ func _triggerNodeDrain(c client.Client, nodeName string, instance *computenodev1
enableLiveMigration = fmt.Sprintf("%v", instance.Spec.Drain.Enabled)

}

for _, nodeToDelete := range instance.Spec.NodesToDelete {
if nodeToDelete.Name == nodeName {
if nodeToDelete.Drain {
enableLiveMigration = fmt.Sprintf("%v", nodeToDelete.Drain)
}
break
}
}
envVars = append(envVars, corev1.EnvVar{Name: "LIVE_MIGRATION_ENABLED", Value: enableLiveMigration})

volumes := []corev1.Volume{
Expand Down Expand Up @@ -971,55 +934,6 @@ func isNodeDrained(c client.Client, nodeName string, jobName string, instance *c
return false, fmt.Errorf("nodeDrain job type %v, reason: %v", conditionsType, conditionsReason)
}

func updateMachineDeletionSelection(c client.Client, instance *computenodev1alpha1.ComputeNodeOpenStack) error {
// We need to delete the old cluster-api-delete-machine labels and add the new ones
nodesToDeleteInfo := make(map[string]string)
for _, newNode := range instance.Spec.NodesToDelete {
nodesToDeleteInfo[newNode.Name] = "Add"
}
for _, oldNode := range instance.Status.NodesToDelete {
_, exists := nodesToDeleteInfo[oldNode.Name]
if exists {
delete(nodesToDeleteInfo, oldNode.Name)
} else {
nodesToDeleteInfo[oldNode.Name] = "Remove"
}
}
machineList := &machinev1beta1.MachineList{}
listOpts := []client.ListOption{
client.InNamespace("openshift-machine-api"),
client.MatchingLabels{"machine.openshift.io/cluster-api-machine-role": instance.Spec.RoleName},
}
err := c.List(context.TODO(), machineList, listOpts...)
if err != nil {
return err
}
annotations := map[string]string{}
for _, machine := range machineList.Items {
if machine.Status.NodeRef == nil {
continue
}

machineNode := machine.Status.NodeRef.Name
action, exists := nodesToDeleteInfo[machineNode]
if !exists {
continue
}

if action == "Add" {
annotations["machine.openshift.io/cluster-api-delete-machine"] = "1"
machine.SetAnnotations(annotations)
} else if action == "Remove" {
annotations["machine.openshift.io/cluster-api-delete-machine"] = "0"
machine.SetAnnotations(annotations)
}
if err := c.Update(context.TODO(), &machine); err != nil {
return err
}
}
return nil
}

func ensureMachineSetSync(c client.Client, instance *computenodev1alpha1.ComputeNodeOpenStack) error {
// get replicas at the openshift-machine-api machineset
workerMachineSet := &machinev1beta1.MachineSet{}
Expand Down Expand Up @@ -1202,7 +1116,7 @@ func deleteAllBlockerPodFinalizer(r *ComputeNodeOpenStackReconciler, instance *c
/* Steps to delete the drain the node
1. Predraining: disable nova service, (optional) migrate VMs, wait until there is no VMs
2. Postdraining: taint the node, remove nova-compute from nova services and placement
3. Remove cleanup jobs, blocker pod finalizer, and update nodesToDelete status information
3. Remove cleanup jobs, blocker pod finalizer
*/
// 1. NodePreDrain
allNodesPreDrained := true
Expand Down