Feature/karpenter

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=$BUILDPLATFORM golang:1.22-alpine as builder
+FROM --platform=$BUILDPLATFORM golang:1.22-alpine AS builder
 
 ARG TARGETOS
 ARG TARGETARCH

cmd/k8s-node-label/main.go

@@ -34,6 +34,7 @@ func main() {
 	leaseLockName := flag.String("lease-lock-name", "k8s-node-label", "Lease lock resource name")
 	defaultNs := getCurrentNamespace(NamespaceFile)
 	leaseLockNamespace := flag.String("lease-lock-namespace", defaultNs, "Lease lock resource namespace")
+	karpenterEnabled := flag.Bool("karpenter-enabled", true, "Karpenter-managed nodes labeled with karpenter.sh/nodepool will be also labeled with node-role.kubernetes.io/karpenter")
 
 	flag.Parse()
 
@@ -90,7 +91,7 @@ func main() {
 		Callbacks: leaderelection.LeaderCallbacks{
 			OnStartedLeading: func(ctx context.Context) {
 				log.Infof("Starting workload as lead: %s", *leaseId)
-				controller.NewNodeController(client, spotProvider, *excludeNodeFromLoadbalancer, *alphaFlags, *excludeEviction, *controlPlaneTaint, *controlPlaneLegacyLabel, *customRoleLabel).Controller.Run(wait.NeverStop)
+				controller.NewNodeController(client, spotProvider, *excludeNodeFromLoadbalancer, *alphaFlags, *excludeEviction, *controlPlaneTaint, *controlPlaneLegacyLabel, *customRoleLabel, *karpenterEnabled).Controller.Run(wait.NeverStop)
 			},
 			OnStoppedLeading: func() {
 				// we can do cleanup here

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/aws/aws-sdk-go v1.54.6
 	github.com/google/uuid v1.6.0
 	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	k8s.io/api v0.29.6
 	k8s.io/apimachinery v0.29.6
 	k8s.io/client-go v0.29.6

go.sum

@@ -78,8 +78,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

pkg/controller/controller.go

@@ -25,6 +25,7 @@ type NodeController struct {
 	controlPlaneTaint       string
 	controlPlaneLegacyLabel bool
 	customRoleLabel         string
+	karpenterEnabled        bool
 }
 
 const (
@@ -38,9 +39,11 @@ const (
 	NodeRoleWorkerLabel           = "node-role.kubernetes.io/worker"
 	NodeRoleSpotWorkerLabel       = "node-role.kubernetes.io/spot-worker"
 	NodeUninitialziedTaint        = "node.cloudprovider.kubernetes.io/uninitialized"
+	NodeKarpenterManagedLabelKey  = "karpenter.sh/nodepool"
+	NodeKarpenterLabel            = "node-role.kubernetes.io/karpenter"
 )
 
-func NewNodeController(client kubernetes.Interface, spotInstanceDiscovery spotdiscovery.SpotDiscoveryInterface, excludeLoadBalancing bool, includeAlphaLabel bool, excludeEviction bool, controlPlaneTaint string, controlPlaneLegacyLabel bool, customRoleLabel string) NodeController {
+func NewNodeController(client kubernetes.Interface, spotInstanceDiscovery spotdiscovery.SpotDiscoveryInterface, excludeLoadBalancing bool, includeAlphaLabel bool, excludeEviction bool, controlPlaneTaint string, controlPlaneLegacyLabel bool, customRoleLabel string, karpenterEnabled bool) NodeController {
 	c := NodeController{
 		client:                  client,
 		includeAlphaLabel:       includeAlphaLabel,
@@ -50,6 +53,7 @@ func NewNodeController(client kubernetes.Interface, spotInstanceDiscovery spotdi
 		controlPlaneTaint:       controlPlaneTaint,
 		controlPlaneLegacyLabel: controlPlaneLegacyLabel,
 		customRoleLabel:         customRoleLabel,
+		karpenterEnabled:        karpenterEnabled,
 	}
 
 	nodeListWatcher := cache.NewListWatchFromClient(
@@ -114,6 +118,12 @@ func (c NodeController) markNode(node *v1.Node) {
 		}
 	}
 
+	if c.karpenterEnabled && isNodeManagedByKarpenter(node) && !isAlreadyMarkedKarpenterNode(node) {
+		log.Infof("Mark node %s with karpenter role label %s", node.Name, NodeKarpenterLabel)
+		addKarpenterLabel(nodeCopy)
+		nodeChanged = true
+	}
+
 	if nodeChanged {
 		_, err := c.client.CoreV1().Nodes().Update(context.TODO(), nodeCopy, metav1.UpdateOptions{})
 		if err != nil {
@@ -251,3 +261,25 @@ func (c NodeController) isNodeInitialized(node *v1.Node) bool {
 	}
 	return true
 }
+
+func isNodeManagedByKarpenter(node *v1.Node) bool {
+	if node.Labels == nil {
+		return false
+	}
+	_, ok := node.Labels[NodeKarpenterManagedLabelKey]
+
+	return ok
+}
+
+func addKarpenterLabel(node *v1.Node) {
+	node.Labels[NodeKarpenterLabel] = ""
+}
+
+func isAlreadyMarkedKarpenterNode(node *v1.Node) bool {
+	if node.Labels == nil {
+		return false
+	}
+
+	_, ok := node.Labels[NodeKarpenterLabel]
+	return ok
+}

pkg/controller/controller_test.go

@@ -74,6 +74,19 @@ var WorkerNode = &v1.Node{
 		ProviderID: "aws:///eu-central-1/i-123qwe123",
 	},
 }
+
+var KarpenterWorkerNode = &v1.Node{
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "test-worker-node",
+		Labels: map[string]string{
+			NodeKarpenterManagedLabelKey: "some-pool",
+		},
+	},
+	Spec: v1.NodeSpec{
+		ProviderID: "aws:///eu-central-1/i-123qwe123",
+	},
+}
+
 var WorkerNodeWithCustomLabel = &v1.Node{
 	ObjectMeta: metav1.ObjectMeta{
 		Name: "test-worker-node-with-label",
@@ -124,7 +137,7 @@ func TestHandlerShouldSetNodeRoleMasterAndControlPlaneForMaster(t *testing.T) {
 	clientset := fake.NewSimpleClientset(MasterNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleMasterLabel, true, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleMasterLabel, true, "", false)
 	c.handler(MasterNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-master-node", metav1.GetOptions{})
@@ -140,7 +153,7 @@ func TestHandlerShouldSetSpotMasterRoleAndControlPlaneForMaster(t *testing.T) {
 	clientset := fake.NewSimpleClientset(SoptMasterNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleMasterLabel, true, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleMasterLabel, true, "", false)
 	c.handler(SoptMasterNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-spot-master", metav1.GetOptions{})
@@ -156,7 +169,7 @@ func TestHandlerShouldSetNodeRoleMasterAndControlPlaneForControlPlane(t *testing
 	clientset := fake.NewSimpleClientset(ControlPlaneNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, true, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, true, "", false)
 	c.handler(ControlPlaneNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-control-plane-node", metav1.GetOptions{})
@@ -172,7 +185,7 @@ func TestHandlerShouldSetSpotMasterRoleAndControlPlaneForControlPlane(t *testing
 	clientset := fake.NewSimpleClientset(SoptControlPlaneNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, true, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, true, "", false)
 	c.handler(SoptControlPlaneNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-spot-control-plane-node", metav1.GetOptions{})
@@ -188,7 +201,7 @@ func TestHandlerShouldSetNodeRoleControlPlane(t *testing.T) {
 	clientset := fake.NewSimpleClientset(ControlPlaneNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "", false)
 	c.handler(ControlPlaneNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-control-plane-node", metav1.GetOptions{})
@@ -204,7 +217,7 @@ func TestHandlerShouldSetSpotControlPlane(t *testing.T) {
 	clientset := fake.NewSimpleClientset(SoptControlPlaneNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "", false)
 	c.handler(SoptControlPlaneNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-spot-control-plane-node", metav1.GetOptions{})
@@ -220,7 +233,7 @@ func TestHandlerShouldSetWorkerRoleIfWorker(t *testing.T) {
 	clientset := fake.NewSimpleClientset(WorkerNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "", false)
 	c.handler(WorkerNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-worker-node", metav1.GetOptions{})
@@ -236,7 +249,7 @@ func TestHandlerShouldSetSpotWorkerRoleIfSpotWorker(t *testing.T) {
 	clientset := fake.NewSimpleClientset(SpotWorkerNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "", false)
 	c.handler(SpotWorkerNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-spot-node", metav1.GetOptions{})
@@ -255,7 +268,7 @@ func TestHandlerShouldSetWorkerRoleIfNotSet(t *testing.T) {
 	clientset := fake.NewSimpleClientset(UnManagedNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "", false)
 	c.handler(UnManagedNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-unmanaged-node", metav1.GetOptions{})
@@ -268,7 +281,7 @@ func TestHandlerShouldPreventMasterFromLoadbalancing(t *testing.T) {
 	clientset := fake.NewSimpleClientset(MasterNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, true, true, false, NodeRoleMasterLabel, true, "")
+	c := NewNodeController(clientset, testingMockDiscovery, true, true, false, NodeRoleMasterLabel, true, "", false)
 	c.handler(MasterNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-master-node", metav1.GetOptions{})
@@ -293,7 +306,7 @@ func TestHandlerShouldPreventControlPlaneFromLoadbalancing(t *testing.T) {
 	clientset := fake.NewSimpleClientset(ControlPlaneNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, true, true, false, NodeRoleControlPlaneLabel, false, "")
+	c := NewNodeController(clientset, testingMockDiscovery, true, true, false, NodeRoleControlPlaneLabel, false, "", false)
 	c.handler(ControlPlaneNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-control-plane-node", metav1.GetOptions{})
@@ -318,7 +331,7 @@ func TestHandlerShouldExcludeNodeFromEviction(t *testing.T) {
 	clientset := fake.NewSimpleClientset(ControlPlaneNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, true, NodeRoleControlPlaneLabel, false, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, true, NodeRoleControlPlaneLabel, false, "", false)
 	c.handler(ControlPlaneNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-control-plane-node", metav1.GetOptions{})
@@ -347,7 +360,7 @@ func TestCustomRoleLabelFound(t *testing.T) {
 	customRoleLabel := "customLabel"
 	expectedRole := "customRole"
 	var expectedErr error = nil
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, true, NodeRoleControlPlaneLabel, false, customRoleLabel)
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, true, NodeRoleControlPlaneLabel, false, customRoleLabel, false)
 	role, err := c.getCustomRoleLabelValue(WorkerNodeWithCustomLabel)
 
 	assert.Equalf(t, expectedRole, role, "Role %s is not equal to %s", role, expectedRole)
@@ -360,7 +373,7 @@ func TestCustomRoleLabelNotFound(t *testing.T) {
 	customRoleLabel := "customLabel"
 	expectedRole := ""
 	expectedErr := fmt.Errorf("Node %s doesn't have %s label", WorkerNode.Name, customRoleLabel)
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, true, NodeRoleControlPlaneLabel, false, customRoleLabel)
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, true, NodeRoleControlPlaneLabel, false, customRoleLabel, false)
 	role, err := c.getCustomRoleLabelValue(WorkerNode)
 
 	assert.Equalf(t, expectedRole, role, "Role %s is not equal to %s", role, expectedRole)
@@ -373,7 +386,7 @@ func TestHandlerShouldSetCustomRoleIfLabelPresent(t *testing.T) {
 	clientset := fake.NewSimpleClientset(WorkerNodeWithCustomLabel)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "customLabel")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "customLabel", false)
 	c.handler(WorkerNodeWithCustomLabel)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-worker-node-with-label", metav1.GetOptions{})
@@ -386,7 +399,7 @@ func TestHandlerShouldNotSetCustomRoleIfLabelNotPresent(t *testing.T) {
 	clientset := fake.NewSimpleClientset(WorkerNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "customLabel")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "customLabel", false)
 	c.handler(WorkerNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-worker-node", metav1.GetOptions{})
@@ -417,7 +430,7 @@ func TestIsNodeInitializedTrue(t *testing.T) {
 	}
 	clientset := fake.NewSimpleClientset(initializedNode)
 	testingMockDiscovery := TestingMockDiscovery{}
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "", false)
 
 	result := c.isNodeInitialized(initializedNode)
 	expected := true
@@ -430,7 +443,7 @@ func TestIsNodeInitializedFalse(t *testing.T) {
 
 	clientset := fake.NewSimpleClientset(UninitializedNode)
 	testingMockDiscovery := TestingMockDiscovery{}
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "", false)
 
 	result := c.isNodeInitialized(UninitializedNode)
 	expected := false
@@ -443,7 +456,7 @@ func TestHandlerShouldNotSetLabelIfNodeUninitialized(t *testing.T) {
 	clientset := fake.NewSimpleClientset(UninitializedNode)
 	testingMockDiscovery := TestingMockDiscovery{}
 
-	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "")
+	c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "", false)
 	c.handler(UninitializedNode)
 
 	foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), "test-uninitialized-control-plane-node", metav1.GetOptions{})
@@ -453,3 +466,60 @@ func TestHandlerShouldNotSetLabelIfNodeUninitialized(t *testing.T) {
 		}
 	}
 }
+
+func TestKubernetesNodeLabeling(t *testing.T) {
+	testCases := []struct {
+		name             string
+		karpenterEnabled bool
+		node             *v1.Node
+		expectedLabels   map[string]string
+	}{
+		{
+			name:             "karpenter disabled for karpenter node",
+			karpenterEnabled: false,
+			node:             KarpenterWorkerNode,
+			expectedLabels: map[string]string{
+				"karpenter.sh/nodepool":          "some-pool",
+				"node-role.kubernetes.io/worker": "",
+			},
+		},
+		{
+			name:             "karpenter enabled for karpenter node",
+			karpenterEnabled: true,
+			node:             KarpenterWorkerNode,
+			expectedLabels: map[string]string{
+				"karpenter.sh/nodepool":             "some-pool",
+				"node-role.kubernetes.io/worker":    "",
+				"node-role.kubernetes.io/karpenter": "",
+			},
+		},
+		{
+			name:             "karpenter enabled for non-karpenter node",
+			karpenterEnabled: true,
+			node:             WorkerNode,
+			expectedLabels: map[string]string{
+				"node-role.kubernetes.io/worker": "",
+			},
+		},
+		{
+			name:             "karpenter enabled for control-plane node",
+			karpenterEnabled: true,
+			node:             ControlPlaneNode,
+			expectedLabels: map[string]string{
+				"node-role.kubernetes.io/control-plane": "",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			clientset := fake.NewSimpleClientset(tc.node)
+			testingMockDiscovery := TestingMockDiscovery{}
+			c := NewNodeController(clientset, testingMockDiscovery, false, false, false, NodeRoleControlPlaneLabel, false, "", tc.karpenterEnabled)
+			c.handler(tc.node)
+
+			foundNode, _ := clientset.CoreV1().Nodes().Get(context.TODO(), tc.node.Name, metav1.GetOptions{})
+			assert.Equal(t, tc.expectedLabels, foundNode.Labels)
+		})
+	}
+}