From 7b77023931da0021f9d2ef1ab0b0e4645adc82aa Mon Sep 17 00:00:00 2001 From: Jiaxin Shan Date: Fri, 15 Nov 2024 01:03:16 -0800 Subject: [PATCH 1/2] Add scheduler config into the vineyardd api Update crd manifests Since I used higher level controller-gen, it introduces additional changes, I do not want to fallback to lower version because our cluster is actually in v1.28 which requires higher version. --- k8s/apis/k8s/v1alpha1/vineyardd_types.go | 25 ++ .../k8s/v1alpha1/zz_generated.deepcopy.go | 35 +++ k8s/config/crd/bases/k8s.v6d.io_backups.yaml | 8 +- .../crd/bases/k8s.v6d.io_csidrivers.yaml | 8 +- .../crd/bases/k8s.v6d.io_globalobjects.yaml | 8 +- .../crd/bases/k8s.v6d.io_localobjects.yaml | 8 +- .../crd/bases/k8s.v6d.io_operations.yaml | 8 +- k8s/config/crd/bases/k8s.v6d.io_recovers.yaml | 8 +- k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml | 8 +- .../crd/bases/k8s.v6d.io_vineyardds.yaml | 295 +++++++++++++++++- 10 files changed, 355 insertions(+), 56 deletions(-) diff --git a/k8s/apis/k8s/v1alpha1/vineyardd_types.go b/k8s/apis/k8s/v1alpha1/vineyardd_types.go index feb91de7b..0101432ae 100644 --- a/k8s/apis/k8s/v1alpha1/vineyardd_types.go +++ b/k8s/apis/k8s/v1alpha1/vineyardd_types.go @@ -101,6 +101,26 @@ type VolumeConfig struct { MountPath string `json:"mountPath,omitempty"` } +// SchedulingConfig holds all configurations about scheduling +type SchedulingConfig struct { + // NodeName is a request to schedule this pod onto a specific node. If it is non-empty, + // the scheduler simply schedules this pod onto that node, assuming that it fits resource + // requirements. + // +optional + NodeName string `json:"nodeName,omitempty"` + // NodeSelector is a selector which must be true for the pod to fit on a node. + // Selector which must match a node's labels for the pod to be scheduled on that node. + // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + // If specified, the pod's scheduling constraints + // +optional + Affinity *corev1.Affinity `json:"affinity,omitempty"` + // If specified, the pod's tolerations. + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` +} + // VineyardConfig holds all configuration about vineyard container type VineyardConfig struct { // represent the vineyardd's image @@ -228,6 +248,11 @@ type VineyarddSpec struct { // +kubebuilder:default:={pvcName: "", mountPath: ""} Volume VolumeConfig `json:"volume,omitempty"` + // the configuration of scheduling + // +kubebuilder:validation:Optional + // +kubebuilder:default:={} + Scheduler SchedulingConfig `json:"scheduler,omitempty"` + // SecurityContext holds the security context settings for the vineyardd container. // +kubebuilder:validation:Optional // +kubebuilder:default:={} diff --git a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go index 303bd0ca7..42c190c9a 100644 --- a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go +++ b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go @@ -632,6 +632,40 @@ func (in *RecoverStatus) DeepCopy() *RecoverStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulingConfig) DeepCopyInto(out *SchedulingConfig) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulingConfig. +func (in *SchedulingConfig) DeepCopy() *SchedulingConfig { + if in == nil { + return nil + } + out := new(SchedulingConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceConfig) DeepCopyInto(out *ServiceConfig) { *out = *in @@ -877,6 +911,7 @@ func (in *VineyarddSpec) DeepCopyInto(out *VineyarddSpec) { out.PluginImage = in.PluginImage out.Metric = in.Metric out.Volume = in.Volume + in.Scheduler.DeepCopyInto(&out.Scheduler) in.SecurityContext.DeepCopyInto(&out.SecurityContext) if in.Volumes != nil { in, out := &in.Volumes, &out.Volumes diff --git a/k8s/config/crd/bases/k8s.v6d.io_backups.yaml b/k8s/config/crd/bases/k8s.v6d.io_backups.yaml index 8e5d16ca1..25a11593e 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_backups.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: backups.k8s.v6d.io spec: @@ -622,9 +622,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml index 0ac249390..c74d2cc35 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_csidrivers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: csidrivers.k8s.v6d.io spec: @@ -95,9 +95,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_globalobjects.yaml b/k8s/config/crd/bases/k8s.v6d.io_globalobjects.yaml index 62663849d..e24d0984e 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_globalobjects.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_globalobjects.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: globalobjects.k8s.v6d.io spec: @@ -75,9 +75,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_localobjects.yaml b/k8s/config/crd/bases/k8s.v6d.io_localobjects.yaml index 8377720ee..e343b6b0b 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_localobjects.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_localobjects.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: localobjects.k8s.v6d.io spec: @@ -82,9 +82,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_operations.yaml b/k8s/config/crd/bases/k8s.v6d.io_operations.yaml index d47eba6ac..8647003f8 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_operations.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_operations.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: operations.k8s.v6d.io spec: @@ -59,9 +59,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_recovers.yaml b/k8s/config/crd/bases/k8s.v6d.io_recovers.yaml index f5ace4261..eca56a121 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_recovers.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_recovers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: recovers.k8s.v6d.io spec: @@ -53,9 +53,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml b/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml index d2ef7a466..8d403651a 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_sidecars.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: sidecars.k8s.v6d.io spec: @@ -1524,9 +1524,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml b/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml index 4b1ba6654..c7764d0b8 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: vineyardds.k8s.v6d.io spec: @@ -80,6 +80,293 @@ spec: replicas: default: 3 type: integer + scheduler: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + nodeName: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object securityContext: properties: allowPrivilegeEscalation: @@ -1570,9 +1857,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] From b55a54d179c8a8bc9d3517e361992fedbb0b0e1c Mon Sep 17 00:00:00 2001 From: Jiaxin Shan Date: Fri, 15 Nov 2024 14:21:44 -0800 Subject: [PATCH 2/2] Add GPU node affinity and workload affinity to tmpl --- k8s/apis/k8s/v1alpha1/vineyardd_types.go | 15 +- .../k8s/v1alpha1/zz_generated.deepcopy.go | 21 +- k8s/cmd/commands/manager/manager.go | 6 +- .../crd/bases/k8s.v6d.io_vineyardds.yaml | 286 +----------------- k8s/pkg/templates/vineyardd/deployment.yaml | 22 ++ 5 files changed, 33 insertions(+), 317 deletions(-) diff --git a/k8s/apis/k8s/v1alpha1/vineyardd_types.go b/k8s/apis/k8s/v1alpha1/vineyardd_types.go index 0101432ae..377323e7d 100644 --- a/k8s/apis/k8s/v1alpha1/vineyardd_types.go +++ b/k8s/apis/k8s/v1alpha1/vineyardd_types.go @@ -103,22 +103,15 @@ type VolumeConfig struct { // SchedulingConfig holds all configurations about scheduling type SchedulingConfig struct { - // NodeName is a request to schedule this pod onto a specific node. If it is non-empty, + // GPUType is a request to schedule this pod onto a specific GPU node. If it is non-empty, // the scheduler simply schedules this pod onto that node, assuming that it fits resource // requirements. // +optional - NodeName string `json:"nodeName,omitempty"` - // NodeSelector is a selector which must be true for the pod to fit on a node. - // Selector which must match a node's labels for the pod to be scheduled on that node. - // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - // +optional - NodeSelector map[string]string `json:"nodeSelector,omitempty"` + GPUType string `json:"gpuType,omitempty"` // If specified, the pod's scheduling constraints + // value should be the model name, model.aibrix.ai/name: deepseek-coder-7b-instruct // +optional - Affinity *corev1.Affinity `json:"affinity,omitempty"` - // If specified, the pod's tolerations. - // +optional - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + AffinityWorkload string `json:"affinityWorkload,omitempty"` } // VineyardConfig holds all configuration about vineyard container diff --git a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go index 42c190c9a..b6b39676b 100644 --- a/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go +++ b/k8s/apis/k8s/v1alpha1/zz_generated.deepcopy.go @@ -635,25 +635,6 @@ func (in *RecoverStatus) DeepCopy() *RecoverStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SchedulingConfig) DeepCopyInto(out *SchedulingConfig) { *out = *in - if in.NodeSelector != nil { - in, out := &in.NodeSelector, &out.NodeSelector - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.Affinity != nil { - in, out := &in.Affinity, &out.Affinity - *out = new(v1.Affinity) - (*in).DeepCopyInto(*out) - } - if in.Tolerations != nil { - in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulingConfig. @@ -911,7 +892,7 @@ func (in *VineyarddSpec) DeepCopyInto(out *VineyarddSpec) { out.PluginImage = in.PluginImage out.Metric = in.Metric out.Volume = in.Volume - in.Scheduler.DeepCopyInto(&out.Scheduler) + out.Scheduler = in.Scheduler in.SecurityContext.DeepCopyInto(&out.SecurityContext) if in.Volumes != nil { in, out := &in.Volumes, &out.Volumes diff --git a/k8s/cmd/commands/manager/manager.go b/k8s/cmd/commands/manager/manager.go index fcf56022a..d0e8ba89d 100644 --- a/k8s/cmd/commands/manager/manager.go +++ b/k8s/cmd/commands/manager/manager.go @@ -297,7 +297,7 @@ func startScheduler(mgr manager.Manager, schedulerConfigFile string) { } // checkKubernetesVersionForScheduler checks the kubernetes version for the scheduler -// the scheduler only supports kubernetes version 1.19-1.24 +// the scheduler only supports kubernetes version 1.19-1.30 func checkKubernetesVersionForScheduler() (bool, error) { config := util.GetKubernetesConfig() @@ -321,8 +321,8 @@ func checkKubernetesVersionForScheduler() (bool, error) { return false, err } - if major != 1 || minor < 19 || minor > 24 { - return false, fmt.Errorf("the scheduler only supports kubernetes version 1.19-1.24") + if major != 1 || minor < 19 || minor > 30 { + return false, fmt.Errorf("the scheduler only supports kubernetes version 1.19-1.30") } return true, nil diff --git a/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml b/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml index c7764d0b8..3cf3a1a20 100644 --- a/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml +++ b/k8s/config/crd/bases/k8s.v6d.io_vineyardds.yaml @@ -82,290 +82,10 @@ spec: type: integer scheduler: properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - nodeName: + affinityWorkload: + type: string + gpuType: type: string - nodeSelector: - additionalProperties: - type: string - type: object - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array type: object securityContext: properties: diff --git a/k8s/pkg/templates/vineyardd/deployment.yaml b/k8s/pkg/templates/vineyardd/deployment.yaml index 2fc10ff11..522f82f12 100644 --- a/k8s/pkg/templates/vineyardd/deployment.yaml +++ b/k8s/pkg/templates/vineyardd/deployment.yaml @@ -205,6 +205,28 @@ spec: - name: log emptyDir: {} affinity: + {{- if .Spec.Scheduler.GPUType }} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + # extract the key to configuration + - key: machine.cluster.vke.volcengine.com/gpu-name + operator: In + values: + - {{ .Spec.Scheduler.GPUType }} + {{- end }} + {{- if .Spec.Scheduler.AffinityWorkload }} + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "model.aibrix.ai/name" + operator: In + values: + - {{ .Spec.Scheduler.AffinityWorkload }} + topologyKey: "kubernetes.io/hostname" + {{- end }} podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: