diff --git a/internal/controller/controller_utils.go b/internal/controller/controller_utils.go new file mode 100644 index 0000000..76839c1 --- /dev/null +++ b/internal/controller/controller_utils.go @@ -0,0 +1,120 @@ +/* +Copyright 2025 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" + core "k8s.io/api/core/v1" +) + +func generateNodeSelector(cp *v1alpha.ClusterPolicy) map[string]string { + ns := map[string]string{ + "kubernetes.io/arch": "amd64", + } + + if len(cp.Spec.NodeSelector) > 0 { + for k, v := range cp.Spec.NodeSelector { + ns[k] = v + } + } + + if cp.Spec.UseNFDLabeling { + ns["intel.feature.node.kubernetes.io/gpu"] = trueValue + } + + return ns +} + +func generateTolerations(cp *v1alpha.ClusterPolicy) []core.Toleration { + tolerations := []core.Toleration{} + + if len(cp.Spec.Tolerations) > 0 { + tolerations = cp.Spec.Tolerations + } + + return tolerations +} + +func isClusterPolicyBeingDeleted(cp *v1alpha.ClusterPolicy) bool { + // CP is nil, which means the CR was deleted, so we should remove everything. + if cp == nil { + return true + } + + // CP is marked for deletion, so we should remove everything. + if !cp.DeletionTimestamp.IsZero() { + return true + } + + return false +} + +func shouldRemoveDRA(cp *v1alpha.ClusterPolicy) bool { + if isClusterPolicyBeingDeleted(cp) { + return true + } + + // DRA not selected + if cp.Spec.ResourceRegistration != resourceModeDRA { + return true + } + + return false +} + +func shouldRemoveDevicePlugin(cp *v1alpha.ClusterPolicy) bool { + if isClusterPolicyBeingDeleted(cp) { + return true + } + + // DP not selected + if cp.Spec.ResourceRegistration != resourceModeDP { + return true + } + + return false +} + +func shouldRemoveXpumd(cp *v1alpha.ClusterPolicy) bool { + if isClusterPolicyBeingDeleted(cp) { + return true + } + + // No resource monitoring, no xpumd + if !cp.Spec.ResourceMonitoring { + return true + } + + return false +} + +// Convert the integer based log level to a string based log level for the OTel config. +func logLevelForXpum(cp *v1alpha.ClusterPolicy) string { + v := cp.Spec.XpuManagerSpec.LogLevel + v = max(cp.Spec.LogLevel, v) + + switch v { + case 0: + return "error" + case 1: + return "warn" + case 2: + return "info" + default: + return "debug" + } +} diff --git a/internal/controller/deviceplugin_controller.go b/internal/controller/deviceplugin_controller.go index 21588f6..7f850e9 100644 --- a/internal/controller/deviceplugin_controller.go +++ b/internal/controller/deviceplugin_controller.go @@ -26,13 +26,13 @@ import ( core "k8s.io/api/core/v1" "k8s.io/klog/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" "github.com/intel/gpu-base-operator/config/deployments" ) @@ -144,8 +144,18 @@ func dpArgs(spec *v1alpha.ClusterPolicy) []string { return args } +func (r *DevicePluginReconciler) buildDaemonSet(spec *v1alpha.ClusterPolicy) *apps.DaemonSet { + ds := deployments.DevicePluginDaemonset() + r.updateDaemonSetObject(ds, spec) + return ds +} + +func (r *DevicePluginReconciler) buildDaemonSetName(crName string) string { + return fmt.Sprintf("%s-device-plugin", crName) +} + func (r *DevicePluginReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy) { - name := fmt.Sprintf("%s-device-plugin", spec.Name) + name := r.buildDaemonSetName(spec.Name) ds.Name = name ds.Namespace = r.Opts.Namespace @@ -157,25 +167,8 @@ func (r *DevicePluginReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec ds.Spec.Template.Spec.Containers[0].Args = dpArgs(spec) - ds.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/arch": "amd64", - } - - if len(spec.Spec.NodeSelector) > 0 { - for k, v := range spec.Spec.NodeSelector { - ds.Spec.Template.Spec.NodeSelector[k] = v - } - } - - if spec.Spec.UseNFDLabeling { - ds.Spec.Template.Spec.NodeSelector["intel.feature.node.kubernetes.io/gpu"] = trueValue - } - - if len(spec.Spec.Tolerations) > 0 { - ds.Spec.Template.Spec.Tolerations = spec.Spec.Tolerations - } else { - ds.Spec.Template.Spec.Tolerations = nil - } + ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec) + ds.Spec.Template.Spec.Tolerations = generateTolerations(spec) cspec := &ds.Spec.Template.Spec @@ -233,29 +226,7 @@ func (r *DevicePluginReconciler) cleanupOpenShiftResources(ctx context.Context, deleteOpenShiftSCCResources(ctx, r.Client, sccName, roleName, bindingName, saName, r.Opts.Namespace) } -func (r *DevicePluginReconciler) createDaemonSet(ctx context.Context, obj client.Object) (ctrl.Result, error) { - spec := obj.(*v1alpha.ClusterPolicy) - - ds := deployments.DevicePluginDaemonset() - - r.updateDaemonSetObject(ds, spec) - - if err := ctrl.SetControllerReference(obj, ds, r.Scheme); err != nil { - klog.Error(err, "unable to set controller reference") - - return ctrl.Result{}, err - } - - if err := r.Create(ctx, ds); err != nil { - klog.Error(err, "unable to create DaemonSet") - - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func (r *DevicePluginReconciler) removeDeploymentIfExists(ctx context.Context) (ctrl.Result, error) { +func (r *DevicePluginReconciler) removeDeploymentIfExists(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { klog.V(4).Info("Removing Device Plugin deployment") crName := r.Opts.ReqName @@ -264,26 +235,21 @@ func (r *DevicePluginReconciler) removeDeploymentIfExists(ctx context.Context) ( r.cleanupOpenShiftResources(ctx, crName) } - dss := &apps.DaemonSetList{} - labels := client.MatchingLabels{ - appLabel: dpValue, - ownerKey: crName, - } - - if err := r.List(ctx, dss, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") + name := r.buildDaemonSetName(crName) - return ctrl.Result{}, err + ds := &apps.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: r.Opts.Namespace, + }, } - if len(dss.Items) == 0 { - klog.V(4).Info("No DevicePlugin deployment found, nothing to do") - - return ctrl.Result{}, nil + if err := r.Delete(ctx, ds); client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err } - if err := r.Delete(ctx, &dss.Items[0]); err != nil { - return ctrl.Result{}, err + if cp != nil { + cp.Status.DevicePluginStatus = notAvailableStatus } klog.V(4).Info("DevicePlugin deployment removed") @@ -291,24 +257,26 @@ func (r *DevicePluginReconciler) removeDeploymentIfExists(ctx context.Context) ( return ctrl.Result{}, nil } -func (r *DevicePluginReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { - _ = logf.FromContext(ctx) +func (r *DevicePluginReconciler) updateStatus(ctx context.Context, cp *v1alpha.ClusterPolicy) error { + ds := &apps.DaemonSet{} - if cp == nil || !cp.DeletionTimestamp.IsZero() { - return r.removeDeploymentIfExists(ctx) + if err := r.Get(ctx, client.ObjectKey{Name: r.buildDaemonSetName(cp.Name), Namespace: r.Opts.Namespace}, ds); err != nil { + klog.Error(err, "unable to get DP DaemonSet to update status") + + return err } - if cp.Spec.ResourceRegistration != "dp" { - cp.Status.DevicePluginStatus = notAvailableStatus + cp.Status.DevicePluginStatus = fmt.Sprintf("%d/%d", + ds.Status.NumberReady, ds.Status.DesiredNumberScheduled) - return r.removeDeploymentIfExists(ctx) - } + return nil +} - var olderDs apps.DaemonSetList - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), client.MatchingLabels{appLabel: dpValue}); err != nil { - klog.Error(err, "unable to list child DaemonSets") +func (r *DevicePluginReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { + _ = logf.FromContext(ctx) - return ctrl.Result{}, err + if shouldRemoveDevicePlugin(cp) { + return r.removeDeploymentIfExists(ctx, cp) } if r.Opts.OpenShift { @@ -319,36 +287,17 @@ func (r *DevicePluginReconciler) Reconcile(ctx context.Context, cp *v1alpha.Clus } } - if len(olderDs.Items) == 0 { - return r.createDaemonSet(ctx, cp) - } - - // Update DaemonSet - - ds := &olderDs.Items[0] - originalDs := ds.DeepCopy() + ds := r.buildDaemonSet(cp) - r.updateDaemonSetObject(ds, cp) + if _, err := controllerutil.CreateOrPatch(ctx, r.Client, ds, func() error { + r.updateDaemonSetObject(ds, cp) - dsDiff := cmp.Diff(originalDs.Spec.Template.Spec, ds.Spec.Template.Spec, cmpopts.EquateEmpty()) - if len(dsDiff) > 0 { - klog.Info("DS difference", "diff", dsDiff) - - if err := r.Update(ctx, ds); err != nil { - klog.Error(err, "unable to update daemonset", "DaemonSet", ds) - - return ctrl.Result{}, err - } - } - - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), client.MatchingLabels{appLabel: dpValue}); err != nil { - klog.Error(err, "unable to list child DaemonSets") + return nil + }); err != nil { + klog.Error(err, "unable to create or patch DP DaemonSet") return ctrl.Result{}, err } - cp.Status.DevicePluginStatus = fmt.Sprintf("%d/%d", - olderDs.Items[0].Status.NumberReady, olderDs.Items[0].Status.DesiredNumberScheduled) - - return ctrl.Result{}, nil + return ctrl.Result{}, r.updateStatus(ctx, cp) } diff --git a/internal/controller/dra_controller.go b/internal/controller/dra_controller.go index 1f3fee3..310b564 100644 --- a/internal/controller/dra_controller.go +++ b/internal/controller/dra_controller.go @@ -34,10 +34,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" "github.com/intel/gpu-base-operator/config/deployments" ) @@ -57,7 +56,7 @@ const ( draResourcePart = "gpu-dra" ) -func (r *DRAReconciler) createAll(ctx context.Context, cp *v1alpha.ClusterPolicy) error { +func (r *DRAReconciler) createDependencyComponentsIfMissing(ctx context.Context, cp *v1alpha.ClusterPolicy) error { objects := []client.Object{} objName := fmt.Sprintf("%s-gpu-dra", cp.Name) @@ -107,7 +106,7 @@ func (r *DRAReconciler) createAll(ctx context.Context, cp *v1alpha.ClusterPolicy for _, o := range objects { if err := r.Create(ctx, o); err != nil { if errors.IsAlreadyExists(err) { - klog.Info("object already exists: "+o.GetName(), o) + klog.V(4).Infof("object already exists: %s (%+v)", o.GetName(), o.GetObjectKind().GroupVersionKind()) continue } @@ -118,38 +117,23 @@ func (r *DRAReconciler) createAll(ctx context.Context, cp *v1alpha.ClusterPolicy } } - return r.createDaemonSet(ctx, cp) + return nil } func (r *DRAReconciler) ensureVfioDeviceClass(ctx context.Context, manageBinding bool) { - existing := deployments.DynamicResourceAllocationDeviceClassVfio(!manageBinding) - - if err := r.Get(ctx, client.ObjectKey{Name: existing.Name}, existing); err != nil { - if errors.IsNotFound(err) { - klog.Info("VFIO device class not found, creating") - - if err := r.Create(ctx, existing); err != nil { - klog.Error(err, "unable to create VFIO device class") - } - - return - } - - klog.Error(err, "unable to get VFIO device class") - return - } - desired := deployments.DynamicResourceAllocationDeviceClassVfio(!manageBinding) - diff := cmp.Diff(existing.Spec.Selectors, desired.Spec.Selectors, cmpopts.EquateEmpty()) - if len(diff) > 0 { - klog.V(2).Info("Updating VFIO device class due to ManageBinding change", "diff", diff) + selectors := desired.Spec.Selectors - existing.Spec.Selectors = desired.Spec.Selectors + if ret, err := controllerutil.CreateOrPatch(ctx, r.Client, desired, func() error { + desired.Spec.Selectors = selectors - if err := r.Update(ctx, existing); err != nil { - klog.Error(err, "unable to update VFIO device class") - } + return nil + }); err != nil { + klog.Error(err, "unable to create or patch VFIO device class") + return + } else { + klog.V(4).Infof("VFIO device class %s %s", desired.Name, ret) } } @@ -357,8 +341,12 @@ func removeHealthCheckIfExists(container *core.Container) { container.LivenessProbe = nil } +func (r *DRAReconciler) buildDaemonSetName(crName string) string { + return fmt.Sprintf("%s-gpu-dra", crName) +} + func (r *DRAReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy) { - name := fmt.Sprintf("%s-gpu-dra", spec.Name) + name := r.buildDaemonSetName(spec.Name) ds.Name = name ds.Namespace = r.Opts.Namespace @@ -370,26 +358,8 @@ func (r *DRAReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v1alpha. ds.Spec.Template.Spec.Containers[0].Image = dspec.Image ds.Spec.Template.Spec.Containers[0].Args = r.generateArgs(spec) - - ds.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/arch": "amd64", - } - - if len(spec.Spec.NodeSelector) > 0 { - for k, v := range spec.Spec.NodeSelector { - ds.Spec.Template.Spec.NodeSelector[k] = v - } - } - - if spec.Spec.UseNFDLabeling { - ds.Spec.Template.Spec.NodeSelector["intel.feature.node.kubernetes.io/gpu"] = trueValue - } - - if len(spec.Spec.Tolerations) > 0 { - ds.Spec.Template.Spec.Tolerations = spec.Spec.Tolerations - } else { - ds.Spec.Template.Spec.Tolerations = nil - } + ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec) + ds.Spec.Template.Spec.Tolerations = generateTolerations(spec) cspec := &ds.Spec.Template.Spec @@ -428,39 +398,29 @@ func (r *DRAReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v1alpha. } } -func (r *DRAReconciler) createDaemonSet(ctx context.Context, spec *v1alpha.ClusterPolicy) error { +func (r *DRAReconciler) buildDraDaemonset(spec *v1alpha.ClusterPolicy) *apps.DaemonSet { ds := deployments.DynamicResourceAllocationDaemonset() r.updateDaemonSetObject(ds, spec) - if err := r.Create(ctx, ds); err != nil { - klog.Error(err, "unable to create DaemonSet") - - return err - } - - return nil + return ds } -func (r *DRAReconciler) removeDeploymentIfExists(ctx context.Context) (ctrl.Result, error) { - crName := r.Opts.ReqName +func (r *DRAReconciler) updateStatus(ctx context.Context, cp *v1alpha.ClusterPolicy) error { + ds := &apps.DaemonSet{} + if err := r.Get(ctx, client.ObjectKey{Name: r.buildDaemonSetName(cp.Name), Namespace: r.Opts.Namespace}, ds); err != nil { + klog.Error(err, "unable to get DRA DaemonSet to update status") - dss := &apps.DaemonSetList{} - labels := client.MatchingLabels{ - appLabel: draValue, - ownerKey: crName, + return err } - if err := r.List(ctx, dss, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err - } + cp.Status.DRAStatus = fmt.Sprintf("%d/%d", + ds.Status.NumberReady, ds.Status.DesiredNumberScheduled) - if len(dss.Items) == 0 { - return ctrl.Result{}, nil - } + return nil +} +func (r *DRAReconciler) removeDeploymentIfExists(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { // If there are any allocated ResourceClaims, removal of DRA will cause // the Pods using them to be stuck at Terminating. // Requeue and try again later. @@ -472,6 +432,10 @@ func (r *DRAReconciler) removeDeploymentIfExists(ctx context.Context) (ctrl.Resu return ctrl.Result{}, err } + if cp != nil { + cp.Status.DRAStatus = notAvailableStatus + } + klog.Info("DRA deployment removed") return ctrl.Result{}, nil @@ -517,24 +481,8 @@ func (r *DRAReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy return ctrl.Result{}, nil } - if cp == nil || !cp.DeletionTimestamp.IsZero() { - return r.removeDeploymentIfExists(ctx) - } - - // DRA not selected, remove existing deployment if exists - if cp.Spec.ResourceRegistration != resourceModeDRA { - cp.Status.DRAStatus = notAvailableStatus - - return r.removeDeploymentIfExists(ctx) - } - - labels := client.MatchingLabels{appLabel: draValue} - - var olderDs apps.DaemonSetList - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") - - return ctrl.Result{}, err + if shouldRemoveDRA(cp) { + return r.removeDeploymentIfExists(ctx, cp) } if r.Opts.OpenShift { @@ -545,38 +493,26 @@ func (r *DRAReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy } } - if len(olderDs.Items) == 0 { - return ctrl.Result{}, r.createAll(ctx, cp) - } - - // Update DaemonSet - - ds := &olderDs.Items[0] - originalDs := ds.DeepCopy() + if err := r.createDependencyComponentsIfMissing(ctx, cp); err != nil { + klog.Error(err, "unable to create dependency components for DRA") - r.updateDaemonSetObject(ds, cp) - - dsDiff := cmp.Diff(originalDs.Spec.Template.Spec, ds.Spec.Template.Spec, cmpopts.EquateEmpty()) - if len(dsDiff) > 0 { - klog.Info("DRA difference", "diff", dsDiff) - - if err := r.Update(ctx, ds); err != nil { - klog.Error(err, "unable to update daemonset", "DaemonSet", ds) - - return ctrl.Result{}, err - } + return ctrl.Result{}, err } r.ensureVfioDeviceClass(ctx, cp.Spec.DynamicResourceAllocationSpec.ManageBinding) - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), labels); err != nil { - klog.Error(err, "unable to list child DaemonSets") + ds := r.buildDraDaemonset(cp) + + _, err := controllerutil.CreateOrPatch(ctx, r.Client, ds, func() error { + r.updateDaemonSetObject(ds, cp) + + return nil + }) + if err != nil { + klog.Error(err, "unable to create or patch DRA DaemonSet") return ctrl.Result{}, err } - cp.Status.DRAStatus = fmt.Sprintf("%d/%d", - olderDs.Items[0].Status.NumberReady, olderDs.Items[0].Status.DesiredNumberScheduled) - - return ctrl.Result{}, nil + return ctrl.Result{}, r.updateStatus(ctx, cp) } diff --git a/internal/controller/openshift.go b/internal/controller/openshift.go index 74118ed..a84dd1a 100644 --- a/internal/controller/openshift.go +++ b/internal/controller/openshift.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) const ( @@ -195,18 +196,20 @@ func createSCCRole(ctx context.Context, c client.Client, roleName, sccName strin }}, } - existing := &rbac.ClusterRole{} - - err := c.Get(ctx, client.ObjectKey{Name: roleName}, existing) - if errors.IsNotFound(err) { - if createErr := c.Create(ctx, cr); createErr != nil && !errors.IsAlreadyExists(createErr) { - return createErr - } + if _, err := controllerutil.CreateOrPatch(ctx, c, cr, func() error { + cr.Rules = []rbac.PolicyRule{{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{sccName}, + Verbs: []string{"use"}, + }} return nil + }); err != nil { + return fmt.Errorf("failed to create or patch ClusterRole %s: %v", roleName, err) } - return err + return nil } // createSCCRoleBinding creates a ClusterRoleBinding that binds the ServiceAccount to the SCC ClusterRole @@ -226,18 +229,24 @@ func createSCCRoleBinding(ctx context.Context, c client.Client, bindingName, rol }}, } - existing := &rbac.ClusterRoleBinding{} - - err := c.Get(ctx, client.ObjectKey{Name: bindingName}, existing) - if errors.IsNotFound(err) { - if createErr := c.Create(ctx, crb); createErr != nil && !errors.IsAlreadyExists(createErr) { - return createErr + if _, err := controllerutil.CreateOrPatch(ctx, c, crb, func() error { + crb.RoleRef = rbac.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: roleName, } + crb.Subjects = []rbac.Subject{{ + Kind: "ServiceAccount", + Name: saName, + Namespace: namespace, + }} return nil + }); err != nil { + return fmt.Errorf("failed to create or patch ClusterRoleBinding %s: %v", bindingName, err) } - return err + return nil } // createServiceAccount creates a namespace-scoped ServiceAccount if it does not already exist. diff --git a/internal/controller/xpumanager_controller.go b/internal/controller/xpumanager_controller.go index 7adbff4..5f6506d 100644 --- a/internal/controller/xpumanager_controller.go +++ b/internal/controller/xpumanager_controller.go @@ -33,11 +33,10 @@ import ( "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/yaml" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" v1alpha "github.com/intel/gpu-base-operator/api/v1alpha1" "github.com/intel/gpu-base-operator/config/deployments" ) @@ -198,39 +197,6 @@ func processContainerResources(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy, } } -func processNodeSelectors(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy) { - ds.Spec.Template.Spec.NodeSelector = map[string]string{ - "kubernetes.io/arch": "amd64", - } - - if len(spec.Spec.NodeSelector) > 0 { - for k, v := range spec.Spec.NodeSelector { - ds.Spec.Template.Spec.NodeSelector[k] = v - } - } - - if spec.Spec.UseNFDLabeling { - ds.Spec.Template.Spec.NodeSelector["intel.feature.node.kubernetes.io/gpu"] = trueValue - } -} - -// Convert the integer based log level to a string based log level for the OTel config. -func logLevelForXpum(cp *v1alpha.ClusterPolicy) string { - v := cp.Spec.XpuManagerSpec.LogLevel - v = max(cp.Spec.LogLevel, v) - - switch v { - case 0: - return "error" - case 1: - return "warn" - case 2: - return "info" - default: - return "debug" - } -} - // buildOTelConfigData constructs the otel-config.yaml content with thresholds // from the ClusterPolicy health spec applied to the default embedded config. func (r *XpuManagerReconciler) buildOTelConfigData(cp *v1alpha.ClusterPolicy) (string, error) { @@ -379,7 +345,7 @@ func (r *XpuManagerReconciler) createMonitoringResourceClaim(ctx context.Context if err := r.Create(ctx, mct); err != nil { if errors.IsAlreadyExists(err) { - klog.Warning(err, "ResourceClaimTemplate already exists") + klog.V(4).Info(err, "ResourceClaimTemplate already exists") return nil } @@ -421,8 +387,12 @@ func (r *XpuManagerReconciler) fetchConfigHashFromConfigMap(ctx context.Context, return otelConfigHash(overrideCm.Data[otelConfigMapKey]), nil } +func (r *XpuManagerReconciler) buildDaemonSetName(cpName string) string { + return fmt.Sprintf("%s-xpu-manager", cpName) +} + func (r *XpuManagerReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v1alpha.ClusterPolicy, draClaim string, otelConfigMapName string, otelConfigHash string) { - name := fmt.Sprintf("%s-xpu-manager", spec.Name) + name := r.buildDaemonSetName(spec.Name) ds.Name = name ds.Namespace = r.Opts.Namespace @@ -431,20 +401,14 @@ func (r *XpuManagerReconciler) updateDaemonSetObject(ds *apps.DaemonSet, spec *v ownerKey: spec.Name, } - if ds.Spec.Template.Annotations == nil { - ds.Spec.Template.Annotations = map[string]string{} + ds.Spec.Template.Annotations = map[string]string{ + otelConfigHashKey: otelConfigHash, } - ds.Spec.Template.Annotations[otelConfigHashKey] = otelConfigHash processContainerResources(ds, spec, draClaim) processXpumdConfigMapMount(ds, otelConfigMapName) - processNodeSelectors(ds, spec) - - if len(spec.Spec.Tolerations) > 0 { - ds.Spec.Template.Spec.Tolerations = spec.Spec.Tolerations - } else { - ds.Spec.Template.Spec.Tolerations = nil - } + ds.Spec.Template.Spec.NodeSelector = generateNodeSelector(spec) + ds.Spec.Template.Spec.Tolerations = generateTolerations(spec) cspec := &ds.Spec.Template.Spec @@ -507,72 +471,32 @@ func (r *XpuManagerReconciler) cleanupOpenShiftResources(ctx context.Context, cp deleteOpenShiftSCCResources(ctx, r.Client, sccName, roleName, bindingName, saName, r.Opts.Namespace) } -func (r *XpuManagerReconciler) createDaemonSet(ctx context.Context, obj client.Object) (ctrl.Result, error) { - cp := obj.(*v1alpha.ClusterPolicy) - +func (r *XpuManagerReconciler) buildDaemonSet(cp *v1alpha.ClusterPolicy, draClaim, cmName, configHash string) *apps.DaemonSet { ds := deployments.XpuManagerDaemonset() - useDra := r.Opts.DRAEnable && cp.Spec.ResourceRegistration == resourceModeDRA - var draClaim string - - if useDra { - draClaim = cp.Name + "-monitor-claim" - - if err := r.createMonitoringResourceClaim(ctx, obj, draClaim); err != nil { - klog.Error(err, "unable to create ResourceClaimTemplate") - - return ctrl.Result{}, err - } - } - - cmName, configHash, err := r.generateXpumdConfigEntries(ctx, cp) - if err != nil { - klog.Error(err, "unable to create or fetch Xpumd config") - - return ctrl.Result{}, err - } - r.updateDaemonSetObject(ds, cp, draClaim, cmName, configHash) - if err := ctrl.SetControllerReference(obj, ds, r.Scheme); err != nil { - klog.Error(err, "unable to set controller reference") - - return ctrl.Result{}, err - } - - if err := r.Create(ctx, ds); err != nil { - klog.Error(err, "unable to create DaemonSet") - - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil + return ds } -func (r *XpuManagerReconciler) removeDeploymentIfExists(ctx context.Context) (ctrl.Result, error) { +func (r *XpuManagerReconciler) removeDeploymentIfExists(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { if r.Opts.OpenShift { r.cleanupOpenShiftResources(ctx, r.Opts.ReqName) } - dss := &apps.DaemonSetList{} - - matching := client.MatchingLabels{ - xpuLabel: xpuValue, - ownerKey: r.Opts.ReqName, + ds := &apps.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.buildDaemonSetName(r.Opts.ReqName), + Namespace: r.Opts.Namespace, + }, } - if err := r.List(ctx, dss, client.InNamespace(r.Opts.Namespace), matching); err != nil { - klog.Error(err, "unable to list XPU Manager DaemonSets") - + if err := r.Delete(ctx, ds); client.IgnoreNotFound(err) != nil { return ctrl.Result{}, err } - if len(dss.Items) == 0 { - return ctrl.Result{}, nil - } - - if err := r.Delete(ctx, &dss.Items[0]); err != nil { - return ctrl.Result{}, err + if cp != nil { + cp.Status.XPUManagerStatus = notAvailableStatus } klog.Info("XPU Manager deployment removed") @@ -580,25 +504,25 @@ func (r *XpuManagerReconciler) removeDeploymentIfExists(ctx context.Context) (ct return ctrl.Result{}, nil } -func (r *XpuManagerReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { - _ = logf.FromContext(ctx) +func (r *XpuManagerReconciler) updateStatus(ctx context.Context, cp *v1alpha.ClusterPolicy) error { + ds := &apps.DaemonSet{} + if err := r.Get(ctx, client.ObjectKey{Name: r.buildDaemonSetName(cp.Name), Namespace: r.Opts.Namespace}, ds); err != nil { + klog.Error(err, "unable to get XPU Manager DaemonSet to update status") - // Delete DaemonSet if ClusterPolicy is being deleted - if cp == nil || cp.DeletionTimestamp != nil { - return r.removeDeploymentIfExists(ctx) + return err } - if !cp.Spec.ResourceMonitoring { - cp.Status.XPUManagerStatus = notAvailableStatus + cp.Status.XPUManagerStatus = fmt.Sprintf("%d/%d", + ds.Status.NumberReady, ds.Status.DesiredNumberScheduled) - return r.removeDeploymentIfExists(ctx) - } + return nil +} - var olderDs apps.DaemonSetList - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), client.MatchingLabels{xpuLabel: xpuValue}); err != nil { - klog.Error(err, "unable to list child DaemonSets") +func (r *XpuManagerReconciler) Reconcile(ctx context.Context, cp *v1alpha.ClusterPolicy) (ctrl.Result, error) { + _ = logf.FromContext(ctx) - return ctrl.Result{}, err + if shouldRemoveXpumd(cp) { + return r.removeDeploymentIfExists(ctx, cp) } if r.Opts.OpenShift { @@ -609,15 +533,6 @@ func (r *XpuManagerReconciler) Reconcile(ctx context.Context, cp *v1alpha.Cluste } } - if len(olderDs.Items) == 0 { - return r.createDaemonSet(ctx, cp) - } - - // Update DaemonSet - - ds := &olderDs.Items[0] - originalDs := ds.DeepCopy() - useDra := r.Opts.DRAEnable && cp.Spec.ResourceRegistration == resourceModeDRA var draClaim string @@ -639,29 +554,17 @@ func (r *XpuManagerReconciler) Reconcile(ctx context.Context, cp *v1alpha.Cluste return ctrl.Result{}, err } - configMapChanged := originalDs.Spec.Template.Annotations[otelConfigHashKey] != configHash + ds := r.buildDaemonSet(cp, draClaim, cmName, configHash) - r.updateDaemonSetObject(ds, cp, draClaim, cmName, configHash) + if _, err := controllerutil.CreateOrPatch(ctx, r.Client, ds, func() error { + r.updateDaemonSetObject(ds, cp, draClaim, cmName, configHash) - dsDiff := cmp.Diff(originalDs.Spec.Template.Spec, ds.Spec.Template.Spec, cmpopts.EquateEmpty()) - if configMapChanged || len(dsDiff) > 0 { - klog.Info("DS difference", "diff", dsDiff) - - if err := r.Update(ctx, ds); err != nil { - klog.Error(err, "unable to update daemonset", "DaemonSet", ds) - - return ctrl.Result{}, err - } - } - - if err := r.List(ctx, &olderDs, client.InNamespace(r.Opts.Namespace), client.MatchingLabels{xpuLabel: xpuValue}); err != nil { - klog.Error(err, "unable to list child DaemonSets") + return nil + }); err != nil { + klog.Error(err, "unable to create or patch XPU Manager DaemonSet") return ctrl.Result{}, err } - cp.Status.XPUManagerStatus = fmt.Sprintf("%d/%d", - olderDs.Items[0].Status.NumberReady, olderDs.Items[0].Status.DesiredNumberScheduled) - - return ctrl.Result{}, nil + return ctrl.Result{}, r.updateStatus(ctx, cp) }