Prototype integration of SRIOV operator

Dockerfile

@@ -34,8 +34,8 @@ FROM ${OPERATOR_BASE_IMAGE}
 ARG DEST_ROOT=/dest-root
 
 LABEL   com.redhat.component="compute-node-operator-container" \
-        name="cn-osp/compute-node-operator" \
-        version="0.0.1" \
+        name="compute-node-operator" \
+        version="1.0" \
         summary="Compute Node Operator" \
         io.k8s.name="compute-node-operator" \
         io.k8s.description="This image includes the compute-node-operator"

api/v1alpha1/computenodeopenstack_types.go

@@ -95,16 +95,20 @@ type NovaCompute struct {
 
 // NeutronNetwork defines neutron configuration parameters
 type NeutronNetwork struct {
-	Nic              string      `json:"nic"`
-	BridgeMappings   string      `json:"bridgeMappings,omitempty"`
-	MechanishDrivers string      `json:"mechanismDrivers,omitempty"`
-	ServicePlugings  string      `json:"servicePlugins,omitempty"`
-	Sriov            SriovConfig `json:"sriov,omitempty"`
+	Nic              string        `json:"nic"`
+	BridgeMappings   string        `json:"bridgeMappings,omitempty"`
+	MechanishDrivers string        `json:"mechanismDrivers,omitempty"`
+	ServicePlugings  string        `json:"servicePlugins,omitempty"`
+	Sriov            []SriovConfig `json:"sriov,omitempty"`
 }
 
 // SriovConfig defines SRIOV config parameters, such as nic information.
 type SriovConfig struct {
-	DevName string `json:"devName"`
+	Interface  string `json:"interface"`
+	NumVfs     int32  `json:"numVfs,omitempty"`
+	Network    string `json:"network"`
+	DeviceType string `json:"deviceType"`
+	Mtu        int32  `json:"mtu,omitempty"`
 }
 
 // NodeToDelete defines the name of the node to delete and if automatic drain is needed

bindata/worker-osp/009-worker-osp-sriov-network-node-policy.yaml

@@ -0,0 +1,27 @@
+{{ if not (eq .AvailableNodeCount 0) }}
+  {{ $workerOspRole := .WorkerOspRole }}
+  {{ $workerOspRoleFormatted := .WorkerOspRole | replace "-" "_" }}
+  {{ range $i, $e := .Sriov }}
+    {{ if not (eq $i 0) }}
+---
+    {{ end }}
+apiVersion: sriovnetwork.openshift.io/v1
+kind: SriovNetworkNodePolicy
+metadata:
+  name: sriov-policy-{{ $workerOspRole }}-{{ $e.Interface }}
+  namespace: openstack
+  labels:
+    compute-node.openstack.org/sriov-policy: "{{ $workerOspRole }}"
+spec:
+  deviceType: {{ $e.DeviceType }}
+  mtu: {{ $e.Mtu }}
+  nicSelector:
+    pfNames:
+    - {{ $e.Interface }}
+  nodeSelector:
+    node-role.kubernetes.io/{{ $workerOspRole }}: ""
+  numVfs: {{ $e.NumVfs }}
+  priority: 5
+  resourceName: sriovnic_{{ $workerOspRoleFormatted }}_{{ $e.Interface }}
+  {{ end }}
+{{ end }}

config/crd/bases/compute-node.openstack.org_computenodeopenstacks.yaml

@@ -127,14 +127,28 @@ spec:
                 servicePlugins:
                   type: string
                 sriov:
-                  description: SriovConfig defines SRIOV config parameters, such as
-                    nic information.
-                  properties:
-                    devName:
-                      type: string
-                  required:
-                  - devName
-                  type: object
+                  items:
+                    description: SriovConfig defines SRIOV config parameters, such
+                      as nic information.
+                    properties:
+                      deviceType:
+                        type: string
+                      interface:
+                        type: string
+                      mtu:
+                        format: int32
+                        type: integer
+                      network:
+                        type: string
+                      numVfs:
+                        format: int32
+                        type: integer
+                    required:
+                    - deviceType
+                    - interface
+                    - network
+                    type: object
+                  type: array
               required:
               - nic
               type: object

config/rbac/role.yaml

@@ -94,6 +94,7 @@ rules:
 - apiGroups:
   - machine.openshift.io
   - machineconfiguration.openshift.io
+  - sriovnetwork.openshift.io
   resources:
   - '*'
   verbs:

config/samples/compute-node_v1alpha1_computenodeopenstack.yaml

@@ -23,8 +23,13 @@ spec:
     bridgeMappings: "datacentre:br-ex"
     mechanismDrivers: ""
     servicePlugins: ""
-    sriov:
-      devName: "test"
+    sriov: []
+    # Uncomment and customize to allocate VFs on an SRIOV-capable NIC on actual baremetal hardware
+    #- interface: enp7s0
+    #  mtu: 9000
+    #  numVfs: 32
+    #  deviceType: vfio-pci
+    #  network: sriov-phy1
 # needed if openshift-sdn is not running
 #  infraDaemonSets:
 #  - name: multus

controllers/computenodeopenstack_controller.go

@@ -43,6 +43,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	machinev1beta1 "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1"
+	sriovnetworkv1 "github.com/openshift/sriov-network-operator/pkg/apis/sriovnetwork/v1"
 	computenodev1alpha1 "github.com/openstack-k8s-operators/compute-node-operator/api/v1alpha1"
 	bindatautil "github.com/openstack-k8s-operators/compute-node-operator/pkg/bindata_util"
 	computenodeopenstack "github.com/openstack-k8s-operators/compute-node-operator/pkg/computenodeopenstack"
@@ -87,7 +88,7 @@ type ComputeNodeOpenStackReconciler struct {
 // +kubebuilder:rbac:groups=compute-node.openstack.org,resources=deployments/finalizers,verbs=update
 // +kubebuilder:rbac:groups=neutron.openstack.org,resources=ovncontrollers;ovsnodeosps,verbs=create;delete;get;list;patch;update;watch
 // +kubebuilder:rbac:groups=nova.openstack.org,resources=novacomputes;virtlogds;libvirtds;iscsids;novamigrationtargets,verbs=create;delete;get;list;patch;update;watch
-// +kubebuilder:rbac:groups=machine.openshift.io;machineconfiguration.openshift.io,resources="*",verbs="*"
+// +kubebuilder:rbac:groups=machine.openshift.io;machineconfiguration.openshift.io;sriovnetwork.openshift.io,resources="*",verbs="*"
 // +kubebuilder:rbac:groups=core,namespace=openstack,resources=pods,verbs=create;delete;get;list;patch;update;watch
 // +kubebuilder:rbac:groups=security.openshift.io,namespace=openstack,resources=securitycontextconstraints,resourceNames=hostnetwork,verbs=use
 
@@ -177,12 +178,15 @@ func (r *ComputeNodeOpenStackReconciler) Reconcile(req ctrl.Request) (ctrl.Resul
 		return ctrl.Result{}, err
 	}
 
+	// Available nodes in the role's machineset
+	var nodeCount int32
+
 	specMDS, err := util.CalculateHash(instance.Spec)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
 	if reflect.DeepEqual(specMDS, instance.Status.SpecMDS) {
-		err = ensureMachineSetSync(r.Client, instance)
+		nodeCount, err = ensureMachineSetSync(r.Client, instance)
 		if err != nil {
 			return ctrl.Result{}, err
 		}
@@ -205,6 +209,19 @@ func (r *ComputeNodeOpenStackReconciler) Reconcile(req ctrl.Request) (ctrl.Resul
 		return ctrl.Result{}, err
 	}
 
+	// Needed for SRIOV considerations in templates, because you do not want to attempt creating
+	// SriovNetworkNodePolicy resources if there are no nodes available in this role's machineset.
+	// Otherwise the SRIOV operator will throw an error when trying to apply any SriovNetworkNodePolicy
+	// that targets the nodes with the role (as it will find no nodes available, which it considers
+	// an error)
+	data.Data["AvailableNodeCount"] = nodeCount
+
+	// Handle SRIOV removal (if any -- add/update handled by "Worker" template logic below)
+	err = ensureSriovRemovalSync(r.Client, instance)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
 	// if run from image OPERATOR_BINDATA env has the bindata
 	manifestPath, found := os.LookupEnv(manifestEnvVar)
 	if found {
@@ -316,6 +333,35 @@ func (r *ComputeNodeOpenStackReconciler) SetupWithManager(mgr ctrl.Manager) erro
 		return nil
 	})
 
+	// watch for SriovNetworkNodePolicy with compute-node-operator CR owner ref
+	ospSriovNetworkNodePolicyFn := handler.ToRequestsFunc(func(o handler.MapObject) []reconcile.Request {
+		cc := mgr.GetClient()
+		result := []reconcile.Request{}
+		snnp := &sriovnetworkv1.SriovNetworkNodePolicy{}
+		key := client.ObjectKey{Namespace: o.Meta.GetNamespace(), Name: o.Meta.GetName()}
+		err := cc.Get(context.Background(), key, snnp)
+		if err != nil && !errors.IsNotFound(err) {
+			log.Error(err, "Unable to retrieve SriovNetworkNodePolicy %v")
+			return nil
+		}
+
+		label := snnp.ObjectMeta.GetLabels()
+		// verify object has ownerUIDLabelSelector
+		if uid, ok := label[ownerUIDLabelSelector]; ok {
+			log.Info(fmt.Sprintf("SriovNetworkNodePolicy object %s marked with OSP owner ref: %s", o.Meta.GetName(), uid))
+			// return namespace and Name of CR
+			name := client.ObjectKey{
+				Namespace: label[ownerNameSpaceLabelSelector],
+				Name:      label[ownerNameLabelSelector],
+			}
+			result = append(result, reconcile.Request{NamespacedName: name})
+		}
+		if len(result) > 0 {
+			return result
+		}
+		return nil
+	})
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&computenodev1alpha1.ComputeNodeOpenStack{}).
 		Owns(&batchv1.Job{}).
@@ -324,6 +370,10 @@ func (r *ComputeNodeOpenStackReconciler) SetupWithManager(mgr ctrl.Manager) erro
 			&handler.EnqueueRequestsFromMapFunc{
 				ToRequests: ospMachineSetFn,
 			}).
+		Watches(&source.Kind{Type: &sriovnetworkv1.SriovNetworkNodePolicy{}},
+			&handler.EnqueueRequestsFromMapFunc{
+				ToRequests: ospSriovNetworkNodePolicyFn,
+			}).
 		Complete(r)
 }
 
@@ -423,6 +473,9 @@ func getRenderData(ctx context.Context, client client.Client, instance *computen
 	}
 	data.Data["RhcosImageUrl"] = providerData["image"]["url"]
 
+	// Set SriovConfig data
+	data.Data["Sriov"] = instance.Spec.Network.Sriov
+
 	return data, nil
 }
 
@@ -1063,28 +1116,31 @@ func updateMachineDeletionSelection(c client.Client, instance *computenodev1alph
 	return nil
 }
 
-func ensureMachineSetSync(c client.Client, instance *computenodev1alpha1.ComputeNodeOpenStack) error {
-	// get replicas at the openshift-machine-api machineset
+func ensureMachineSetSync(c client.Client, instance *computenodev1alpha1.ComputeNodeOpenStack) (int32, error) {
+	// get replicas at the openshift-machine-api machineset and return the current
+	// available machine replicas if machineset is found
 	workerMachineSet := &machinev1beta1.MachineSet{}
 	machineSetName := instance.Spec.ClusterName + "-" + instance.Spec.RoleName
+	var currentCount int32
 	err := c.Get(context.TODO(), types.NamespacedName{Name: machineSetName, Namespace: "openshift-machine-api"}, workerMachineSet)
 	if err != nil && errors.IsNotFound(err) {
 		log.Info(fmt.Sprintf("Machineset not found, recreate it: %s osp-worker nodes: %d", machineSetName, instance.Spec.Workers))
 		if err := c.Update(context.TODO(), instance); err != nil {
-			return err
+			return currentCount, err
 		}
 	} else if err != nil {
-		return err
+		return currentCount, err
 	} else {
+		currentCount = workerMachineSet.Status.AvailableReplicas
 		if *workerMachineSet.Spec.Replicas != instance.Spec.Workers {
 			// MachineSet has been updated, force CRD re-sync to match the machineset replicas
 			instance.Spec.Workers = *workerMachineSet.Spec.Replicas
 			if err := c.Update(context.TODO(), instance); err != nil {
-				return err
+				return currentCount, err
 			}
 		}
 	}
-	return nil
+	return currentCount, nil
 }
 
 func addToBeRemovedTaint(kclient kubernetes.Interface, node corev1.Node) error {
@@ -1364,6 +1420,56 @@ func deleteAllBlockerPodFinalizer(r *ComputeNodeOpenStackReconciler, instance *c
 	return nil
 }
 
+func ensureSriovRemovalSync(c client.Client, instance *computenodev1alpha1.ComputeNodeOpenStack) error {
+	labelSelectorMap := map[string]string{
+		ownerUIDLabelSelector:       string(instance.UID),
+		ownerNameSpaceLabelSelector: instance.Namespace,
+		ownerNameLabelSelector:      instance.Name,
+	}
+
+	// Get all SRIOV policies tied to this instance
+	sriovNetworkNodePolicies, err := computenodeopenstack.GetSriovNetworkNodePoliciesWithLabel(c, instance, labelSelectorMap, instance.Namespace)
+	if err != nil {
+		return err
+	}
+
+	findInterface := func(instance *computenodev1alpha1.ComputeNodeOpenStack, interfaceName string) bool {
+		for _, sriovConfig := range instance.Spec.Network.Sriov {
+			if sriovConfig.Interface == interfaceName {
+				return true
+			}
+		}
+
+		return false
+	}
+
+	// Iterate through SRIOV policies and delete those that have no interfaces represented
+	// in the instance's Network.Sriov spec
+	for idx := range sriovNetworkNodePolicies.Items {
+		snnp := &sriovNetworkNodePolicies.Items[idx]
+
+		foundInterface := false
+
+		for _, interfaceName := range snnp.Spec.NicSelector.PfNames {
+			foundInterface = findInterface(instance, interfaceName)
+
+			if foundInterface {
+				break
+			}
+		}
+
+		if !foundInterface {
+			err = c.Delete(context.Background(), snnp, &client.DeleteOptions{})
+			if err != nil {
+				return err
+			}
+			log.Info(fmt.Sprintf("Deleting SriovNetworkNodePolicy that does not match instance SRIOV interfaces: name %s - %s", snnp.Name, snnp.UID))
+		}
+	}
+
+	return nil
+}
+
 /* deleteOwnerRefLabeledObjects - cleans up namespaced objects outside the default namespace
    using the owner reference labels added.
    List of objects which get cleaned:
@@ -1372,6 +1478,7 @@ func deleteAllBlockerPodFinalizer(r *ComputeNodeOpenStackReconciler, instance *c
    - user-data secret, openshift-machine-api namespace
    - machineconfigpool, not namespaced
    - machineconfig, not namespaced
+   - sriovnetworknodepolicy, namespace information from CR.Spec
 */
 func deleteOwnerRefLabeledObjects(r *ComputeNodeOpenStackReconciler, instance *computenodev1alpha1.ComputeNodeOpenStack) error {
 	labelSelectorMap := map[string]string{
@@ -1447,5 +1554,20 @@ func deleteOwnerRefLabeledObjects(r *ComputeNodeOpenStackReconciler, instance *c
 		log.Info(fmt.Sprintf("MachineConfig deleted: name %s - %s", mc.Name, mc.UID))
 	}
 
+	// delete sriovnetworknodepolicies in instance namespace
+	sriovNetworkNodePolicies, err := computenodeopenstack.GetSriovNetworkNodePoliciesWithLabel(r.Client, instance, labelSelectorMap, instance.Namespace)
+	if err != nil {
+		return err
+	}
+	for idx := range sriovNetworkNodePolicies.Items {
+		snnp := &sriovNetworkNodePolicies.Items[idx]
+
+		err = r.Client.Delete(context.Background(), snnp, &client.DeleteOptions{})
+		if err != nil {
+			return err
+		}
+		log.Info(fmt.Sprintf("SriovNetworkNodePolicy deleted: name %s - %s", snnp.Name, snnp.UID))
+	}
+
 	return nil
 }