Add control of enable/disable toolbox to the storagecluster controller

This adds the ensureToolbox() function to the StorageClusterReconciler using the new API method, checking the OCSInitialization & the StorageClsuter for the EnableCephTools boolean. If either of them is true the toolbox is created & if both are false toolbox is disabled. Signed-off-by: Malay Kumar Parida <[email protected]>
pdvian · May 16, 2022 · c7a0f89 · c7a0f89
1 parent fc5d99d
commit c7a0f89
Show file tree

Hide file tree

Showing 4 changed files with 282 additions and 0 deletions.
diff --git a/controllers/storagecluster/cephtoolbox.go b/controllers/storagecluster/cephtoolbox.go
@@ -0,0 +1,78 @@
+package storagecluster
+
+import (
+	"context"
+	"reflect"
+
+	ocsv1 "github.com/red-hat-storage/ocs-operator/api/v1"
+	"github.com/red-hat-storage/ocs-operator/controllers/defaults"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+const (
+
+	// RookCephToolDeploymentName is the name of the rook-ceph-tools deployment
+	rookCephToolDeploymentName = "rook-ceph-tools"
+)
+
+func (r *StorageClusterReconciler) ensureToolsDeployment(sc *ocsv1.StorageCluster) error {
+
+	var isFound bool
+	namespace := sc.Namespace
+
+	tolerations := []corev1.Toleration{{
+		Key:      defaults.NodeTolerationKey,
+		Operator: corev1.TolerationOpEqual,
+		Value:    "true",
+		Effect:   corev1.TaintEffectNoSchedule,
+	}}
+
+	tolerations = append(tolerations, sc.Spec.ManagedResources.CephToolbox.Tolerations...)
+
+	toolsDeployment := sc.NewToolsDeployment(tolerations)
+	foundToolsDeployment := &appsv1.Deployment{}
+	err := r.Client.Get(context.TODO(), types.NamespacedName{Name: rookCephToolDeploymentName, Namespace: namespace}, foundToolsDeployment)
+
+	if err == nil {
+		isFound = true
+	} else if errors.IsNotFound(err) {
+		isFound = false
+	} else {
+		return err
+	}
+
+	// Checking spec of ocsinitialization for its Enablecephtools field
+	ocsinit := &ocsv1.OCSInitialization{}
+	err = r.Client.Get(context.TODO(), types.NamespacedName{Name: "ocsinit", Namespace: namespace}, ocsinit)
+	if err != nil && !errors.IsNotFound(err) {
+		return err
+	}
+
+	if sc.Spec.EnableCephTools || ocsinit.Spec.EnableCephTools {
+		// Create or Update if ceph tools is enabled.
+
+		//Adding Ownerreference to the ceph tools
+		err = controllerutil.SetOwnerReference(sc, toolsDeployment, r.Client.Scheme())
+		if err != nil {
+			return err
+		}
+
+		if !isFound {
+			return r.Client.Create(context.TODO(), toolsDeployment)
+		} else if !reflect.DeepEqual(foundToolsDeployment.Spec, toolsDeployment.Spec) {
+
+			updateDeployment := foundToolsDeployment.DeepCopy()
+			updateDeployment.Spec = *toolsDeployment.Spec.DeepCopy()
+
+			return r.Client.Update(context.TODO(), updateDeployment)
+		}
+	} else if isFound {
+		// delete if ceph tools exists and is disabled
+		return r.Client.Delete(context.TODO(), foundToolsDeployment)
+	}
+	return nil
+}
diff --git a/controllers/storagecluster/cephtoolbox_test.go b/controllers/storagecluster/cephtoolbox_test.go
@@ -0,0 +1,191 @@
+package storagecluster
+
+import (
+	"context"
+	"testing"
+
+	v1 "github.com/red-hat-storage/ocs-operator/api/v1"
+	"github.com/red-hat-storage/ocs-operator/controllers/defaults"
+	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+func TestEnsureToolsDeployment(t *testing.T) {
+	testcases := []struct {
+		label           string
+		enableCephTools bool
+		tolerations     []corev1.Toleration
+	}{
+		{
+			label:           "Case 1",
+			enableCephTools: true,
+			tolerations:     []corev1.Toleration{},
+		},
+		{
+			label:           "Case 2",
+			enableCephTools: false,
+			tolerations:     []corev1.Toleration{},
+		},
+		{
+			label:           "Case 3",
+			enableCephTools: true,
+			tolerations: []corev1.Toleration{{
+				Key:      "test-toleration",
+				Operator: corev1.TolerationOpEqual,
+				Value:    "true",
+				Effect:   corev1.TaintEffectNoSchedule,
+			}},
+		},
+	}
+
+	defaultTolerations := []corev1.Toleration{{
+		Key:      defaults.NodeTolerationKey,
+		Operator: corev1.TolerationOpEqual,
+		Value:    "true",
+		Effect:   corev1.TaintEffectNoSchedule,
+	}}
+
+	for _, tc := range testcases {
+		ocs, request, reconciler := getTestParams(false, t)
+		ocs.Spec.EnableCephTools = tc.enableCephTools
+		ocs.Spec.ManagedResources.CephToolbox.Tolerations = tc.tolerations
+
+		err := reconciler.ensureToolsDeployment(&ocs)
+		assert.NoErrorf(t, err, "[%s] failed to create ceph tools", tc.label)
+		if tc.enableCephTools {
+			cephtoolsDeployment := &appsv1.Deployment{}
+			err := reconciler.Client.Get(context.TODO(), types.NamespacedName{Name: rookCephToolDeploymentName, Namespace: request.Namespace}, cephtoolsDeployment)
+			assert.NoErrorf(t, err, "[%s] failed to create ceph tools", tc.label)
+
+			assert.Equalf(
+				t, cephtoolsDeployment.Spec.Template.Spec.Tolerations, append(defaultTolerations, tc.tolerations...),
+				"[%s]: failed to add toleration to the ceph tool deployment resource", tc.label,
+			)
+		}
+	}
+}
+
+func TestEnsureToolsDeploymentUpdate(t *testing.T) {
+	var replicaTwo int32 = 2
+
+	testcases := []struct {
+		label           string
+		enableCephTools bool
+		tolerations     []corev1.Toleration
+	}{
+		{
+			label:           "Case 1", // existing ceph tools pod should get updated
+			enableCephTools: true,
+			tolerations:     []corev1.Toleration{},
+		},
+		{
+			label:           "Case 2", // existing ceph tool pod should get deleted
+			enableCephTools: false,
+		},
+		{
+			label:           "Case 3",
+			enableCephTools: true,
+			tolerations: []corev1.Toleration{{
+				Key:      "test-toleration",
+				Operator: corev1.TolerationOpEqual,
+				Value:    "true",
+				Effect:   corev1.TaintEffectNoSchedule,
+			}},
+		},
+	}
+
+	defaultTolerations := []corev1.Toleration{{
+		Key:      defaults.NodeTolerationKey,
+		Operator: corev1.TolerationOpEqual,
+		Value:    "true",
+		Effect:   corev1.TaintEffectNoSchedule,
+	}}
+
+	for _, tc := range testcases {
+		ocs, request, reconciler := getTestParams(false, t)
+		ocs.Spec.EnableCephTools = tc.enableCephTools
+		ocs.Spec.ManagedResources.CephToolbox.Tolerations = tc.tolerations
+
+		cephTools := &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      rookCephToolDeploymentName,
+				Namespace: request.Namespace,
+			},
+			Spec: appsv1.DeploymentSpec{
+				Replicas: &replicaTwo,
+			},
+		}
+		err := reconciler.Client.Create(context.TODO(), cephTools)
+		assert.NoError(t, err)
+		err = reconciler.ensureToolsDeployment(&ocs)
+		assert.NoErrorf(t, err, "[%s] failed to create ceph tools deployment", tc.label)
+
+		cephtoolsDeployment := &appsv1.Deployment{}
+		err = reconciler.Client.Get(context.TODO(), types.NamespacedName{Name: rookCephToolDeploymentName, Namespace: request.Namespace}, cephtoolsDeployment)
+		if tc.enableCephTools {
+			assert.NoErrorf(t, err, "[%s] failed to get ceph tools deployment", tc.label)
+			assert.Equalf(t, int32(1), *cephtoolsDeployment.Spec.Replicas, "[%s] failed to update the ceph tools pod", tc.label)
+
+			assert.Equalf(
+				t, cephtoolsDeployment.Spec.Template.Spec.Tolerations, append(defaultTolerations, tc.tolerations...),
+				"[%s]: failed to add toleration to the ceph tool deployment resource", tc.label,
+			)
+
+		} else {
+			assert.Errorf(t, err, "[%s] failed to delete ceph tools deployment when it was disabled in the spec", tc.label)
+		}
+	}
+}
+
+func getTestParams(mockNamespace bool, t *testing.T) (v1.StorageCluster, reconcile.Request, StorageClusterReconciler) {
+	var request reconcile.Request
+	if mockNamespace {
+		request = reconcile.Request{
+			NamespacedName: types.NamespacedName{
+				Name:      "test",
+				Namespace: "test-ns",
+			},
+		}
+	} else {
+		request = reconcile.Request{
+			NamespacedName: types.NamespacedName{
+				Name:      request.Name,
+				Namespace: request.Namespace,
+			},
+		}
+	}
+	ocs := v1.StorageCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      request.Name,
+			Namespace: request.Namespace,
+		},
+	}
+
+	reconciler := getTheReconciler(t, &ocs)
+	//The fake client stores the objects after adding a resource version to
+	//them. This is a breaking change introduced in
+	//https://github.com/kubernetes-sigs/controller-runtime/pull/1306.
+	//Therefore we cannot use the fake object that we provided as input to the
+	//the fake client and should use the object obtained from the Get
+	//operation.
+	_ = reconciler.Client.Get(context.TODO(), request.NamespacedName, &ocs)
+
+	return ocs, request, reconciler
+}
+
+func getTheReconciler(t *testing.T, objs ...runtime.Object) StorageClusterReconciler {
+	scheme := createFakeScheme(t)
+	client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objs...).Build()
+
+	return StorageClusterReconciler{
+		Scheme:   scheme,
+		Client:   client,
+		platform: &Platform{},
+	}
+}
diff --git a/controllers/storagecluster/reconcile.go b/controllers/storagecluster/reconcile.go
@@ -160,6 +160,13 @@ func (r *StorageClusterReconciler) Reconcile(ctx context.Context, request reconc
 	// Reconcile changes to the cluster
 	result, reconcileError := r.reconcilePhases(sc, request)
 
+	// Ensure that cephtoolbox is deployed as instructed by the user
+	err := r.ensureToolsDeployment(sc)
+	if err != nil {
+		r.Log.Error(err, "Failed to process ceph tools deployment.", "CephToolDeployment", klog.KRef(sc.Namespace, rookCephToolDeploymentName))
+		return reconcile.Result{}, err
+	}
+
 	// Apply status changes to the storagecluster
 	statusError := r.Client.Status().Update(ctx, sc)
 	if statusError != nil {

diff --git a/controllers/storagecluster/storagecluster_controller.go b/controllers/storagecluster/storagecluster_controller.go
@@ -21,7 +21,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
 var (
@@ -120,5 +122,9 @@ func (r *StorageClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&corev1.PersistentVolumeClaim{}, builder.WithPredicates(pvcPredicate)).
 		Owns(&appsv1.Deployment{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
 		Owns(&corev1.Service{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
+		Watches(
+			&source.Kind{Type: &ocsv1.OCSInitialization{}},
+			&handler.EnqueueRequestForObject{},
+		).
 		Complete(r)
 }