diff --git a/k8s/apis/k8s/v1alpha1/vineyardd_types.go b/k8s/apis/k8s/v1alpha1/vineyardd_types.go index feb91de7b..377323e7d 100644 --- a/k8s/apis/k8s/v1alpha1/vineyardd_types.go +++ b/k8s/apis/k8s/v1alpha1/vineyardd_types.go @@ -101,6 +101,19 @@ type VolumeConfig struct { MountPath string `json:"mountPath,omitempty"` } +// SchedulingConfig holds all configurations about scheduling +type SchedulingConfig struct { + // GPUType is a request to schedule this pod onto a specific GPU node. If it is non-empty, + // the scheduler simply schedules this pod onto that node, assuming that it fits resource + // requirements. + // +optional + GPUType string `json:"gpuType,omitempty"` + // If specified, the pod's scheduling constraints + // value should be the model name, model.aibrix.ai/name: deepseek-coder-7b-instruct + // +optional + AffinityWorkload string `json:"affinityWorkload,omitempty"` +} + // VineyardConfig holds all configuration about vineyard container type VineyardConfig struct { // represent the vineyardd's image @@ -228,6 +241,11 @@ type VineyarddSpec struct { // +kubebuilder:default:={pvcName: "", mountPath: ""} Volume VolumeConfig `json:"volume,omitempty"` + // the configuration of scheduling + // +kubebuilder:validation:Optional + // +kubebuilder:default:={} + Scheduler SchedulingConfig `json:"scheduler,omitempty"` + // SecurityContext holds the security context settings for the vineyardd container. // +kubebuilder:validation:Optional // +kubebuilder:default:={} diff --git a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go index 303bd0ca7..b6b39676b 100644 --- a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go +++ b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go @@ -632,6 +632,21 @@ func (in *RecoverStatus) DeepCopy() *RecoverStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulingConfig) DeepCopyInto(out *SchedulingConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulingConfig. +func (in *SchedulingConfig) DeepCopy() *SchedulingConfig { + if in == nil { + return nil + } + out := new(SchedulingConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceConfig) DeepCopyInto(out *ServiceConfig) { *out = *in @@ -877,6 +892,7 @@ func (in *VineyarddSpec) DeepCopyInto(out *VineyarddSpec) { out.PluginImage = in.PluginImage out.Metric = in.Metric out.Volume = in.Volume + out.Scheduler = in.Scheduler in.SecurityContext.DeepCopyInto(&out.SecurityContext) if in.Volumes != nil { in, out := &in.Volumes, &out.Volumes diff --git a/k8s/cmd/commands/manager/manager.go b/k8s/cmd/commands/manager/manager.go index fcf56022a..d0e8ba89d 100644 --- a/k8s/cmd/commands/manager/manager.go +++ b/k8s/cmd/commands/manager/manager.go @@ -297,7 +297,7 @@ func startScheduler(mgr manager.Manager, schedulerConfigFile string) { } // checkKubernetesVersionForScheduler checks the kubernetes version for the scheduler -// the scheduler only supports kubernetes version 1.19-1.24 +// the scheduler only supports kubernetes version 1.19-1.30 func checkKubernetesVersionForScheduler() (bool, error) { config := util.GetKubernetesConfig() @@ -321,8 +321,8 @@ func checkKubernetesVersionForScheduler() (bool, error) { return false, err } - if major != 1 || minor < 19 || minor > 24 { - return false, fmt.Errorf("the scheduler only supports kubernetes version 1.19-1.24") + if major != 1 || minor < 19 || minor > 30 { + return false, fmt.Errorf("the scheduler only supports kubernetes version 1.19-1.30") } return true, nil diff --git a/k8s/config/crd/bases/k8s.v6d.io_backups.yaml b/k8s/config/crd/bases/k8s.v6d.io_backups.yaml index 8e5d16ca1..25a11593e 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_backups.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: backups.k8s.v6d.io spec: @@ -622,9 +622,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml index 0ac249390..c74d2cc35 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: csidrivers.k8s.v6d.io spec: @@ -95,9 +95,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_globalobjects.yaml b/k8s/config/crd/bases/k8s.v6d.io_globalobjects.yaml index 62663849d..e24d0984e 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_globalobjects.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_globalobjects.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: globalobjects.k8s.v6d.io spec: @@ -75,9 +75,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_localobjects.yaml b/k8s/config/crd/bases/k8s.v6d.io_localobjects.yaml index 8377720ee..e343b6b0b 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_localobjects.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_localobjects.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: localobjects.k8s.v6d.io spec: @@ -82,9 +82,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_operations.yaml b/k8s/config/crd/bases/k8s.v6d.io_operations.yaml index d47eba6ac..8647003f8 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_operations.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_operations.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: operations.k8s.v6d.io spec: @@ -59,9 +59,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_recovers.yaml b/k8s/config/crd/bases/k8s.v6d.io_recovers.yaml index f5ace4261..eca56a121 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_recovers.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_recovers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: recovers.k8s.v6d.io spec: @@ -53,9 +53,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml b/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml index d2ef7a466..8d403651a 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: sidecars.k8s.v6d.io spec: @@ -1524,9 +1524,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml b/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml index 4b1ba6654..3cf3a1a20 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: vineyardds.k8s.v6d.io spec: @@ -80,6 +80,13 @@ spec: replicas: default: 3 type: integer + scheduler: + properties: + affinityWorkload: + type: string + gpuType: + type: string + type: object securityContext: properties: allowPrivilegeEscalation: @@ -1570,9 +1577,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/pkg/templates/vineyardd/deployment.yaml b/k8s/pkg/templates/vineyardd/deployment.yaml index 2fc10ff11..522f82f12 100644 --- a/k8s/pkg/templates/vineyardd/deployment.yaml +++ b/k8s/pkg/templates/vineyardd/deployment.yaml @@ -205,6 +205,28 @@ spec: - name: log emptyDir: {} affinity: + {{- if .Spec.Scheduler.GPUType }} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + # extract the key to configuration + - key: machine.cluster.vke.volcengine.com/gpu-name + operator: In + values: + - {{ .Spec.Scheduler.GPUType }} + {{- end }} + {{- if .Spec.Scheduler.AffinityWorkload }} + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "model.aibrix.ai/name" + operator: In + values: + - {{ .Spec.Scheduler.AffinityWorkload }} + topologyKey: "kubernetes.io/hostname" + {{- end }} podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: