Skip to content

Commit

Permalink
test: add integration tests for crp watcher (#416)
Browse files Browse the repository at this point in the history
  • Loading branch information
zhiying-lin authored Jul 6, 2023
1 parent b6af7aa commit 091ccc8
Show file tree
Hide file tree
Showing 3 changed files with 301 additions and 14 deletions.
102 changes: 102 additions & 0 deletions pkg/controllers/clusterresourceplacementwatcher/suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
/*
Copyright (c) Microsoft Corporation.
Licensed under the MIT license.
*/
package clusterresourceplacementwatcher

import (
"context"
"flag"
"path/filepath"
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
"k8s.io/klog/v2/klogr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/manager"

fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
"go.goms.io/fleet/test/utils/controller"
)

var (
cfg *rest.Config
mgr manager.Manager
k8sClient client.Client
testEnv *envtest.Environment
ctx context.Context
cancel context.CancelFunc
fakePlacementController *controller.FakeController
)

func TestAPIs(t *testing.T) {
RegisterFailHandler(Fail)

RunSpecs(t, "ClusterResourcePlacement Watcher Suite")
}

var _ = BeforeSuite(func() {
klog.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))

ctx, cancel = context.WithCancel(context.TODO())

By("bootstrapping test environment")
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("../../../", "config", "crd", "bases")},
ErrorIfCRDPathMissing: true,
}

var err error
cfg, err = testEnv.Start()
Expect(err).Should(Succeed())
Expect(cfg).NotTo(BeNil())

err = fleetv1beta1.AddToScheme(scheme.Scheme)
Expect(err).NotTo(HaveOccurred())

//+kubebuilder:scaffold:scheme
By("construct the k8s client")
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
Expect(err).Should(Succeed())
Expect(k8sClient).NotTo(BeNil())

By("starting the controller manager")
klog.InitFlags(flag.CommandLine)
flag.Parse()

mgr, err = ctrl.NewManager(cfg, ctrl.Options{
Scheme: scheme.Scheme,
MetricsBindAddress: "0",
Logger: klogr.NewWithOptions(klogr.WithFormat(klogr.FormatKlog)),
})
Expect(err).Should(Succeed())

fakePlacementController = &controller.FakeController{}

err = (&Reconciler{
PlacementController: fakePlacementController,
}).SetupWithManager(mgr)
Expect(err).Should(Succeed())

go func() {
defer GinkgoRecover()
err = mgr.Start(ctx)
Expect(err).Should(Succeed(), "failed to run manager")
}()
})

var _ = AfterSuite(func() {
defer klog.Flush()

cancel()
By("tearing down the test environment")
err := testEnv.Stop()
Expect(err).Should(Succeed())
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
/*
Copyright (c) Microsoft Corporation.
Licensed under the MIT license.
*/
package clusterresourceplacementwatcher

import (
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"

fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
)

const (
testCRPName = "my-crp"
)

func clusterResourcePlacementForTest() *fleetv1beta1.ClusterResourcePlacement {
return &fleetv1beta1.ClusterResourcePlacement{
ObjectMeta: metav1.ObjectMeta{
Name: testCRPName,
},
Spec: fleetv1beta1.ClusterResourcePlacementSpec{
ResourceSelectors: []fleetv1beta1.ClusterResourceSelector{
{
Group: corev1.GroupName,
Version: "v1",
Kind: "Service",
LabelSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"region": "east"},
},
},
},
Policy: &fleetv1beta1.PlacementPolicy{},
},
}
}

var _ = Describe("Test ClusterResourcePlacement Watcher", func() {
const (
eventuallyTimeout = time.Second * 10
consistentlyDuration = time.Second * 10
interval = time.Millisecond * 250
)

var (
createdCRP = &fleetv1beta1.ClusterResourcePlacement{}
)

BeforeEach(func() {
fakePlacementController.ResetQueue()
By("By creating a new clusterResourcePlacement")
crp := clusterResourcePlacementForTest()
Expect(k8sClient.Create(ctx, crp)).Should(Succeed())
})

Context("When creating new clusterResourcePlacement", func() {
AfterEach(func() {
By("By deleting crp")
createdCRP = clusterResourcePlacementForTest()
Expect(k8sClient.Delete(ctx, createdCRP)).Should(Succeed())

By("By checking crp")
Eventually(func() bool {
return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testCRPName}, createdCRP))
}, eventuallyTimeout, interval).Should(BeTrue(), "crp should be deleted")
})

It("Should enqueue the event", func() {
By("By checking placement controller queue")
Eventually(func() bool {
return fakePlacementController.Key() == testCRPName
}, eventuallyTimeout, interval).Should(BeTrue(), "placementController should receive the CRP name")
})
})

Context("When updating clusterResourcePlacement", func() {
BeforeEach(func() {
By("By resetting the placement queue")
fakePlacementController.ResetQueue()
})
AfterEach(func() {
By("By deleting crp")
createdCRP = clusterResourcePlacementForTest()
Expect(k8sClient.Delete(ctx, createdCRP)).Should(Succeed())

By("By checking crp")
Eventually(func() bool {
return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testCRPName}, createdCRP))
}, eventuallyTimeout, interval).Should(BeTrue(), "crp should be deleted")
})

It("Updating the spec and it should enqueue the event", func() {
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: testCRPName}, createdCRP)).Should(Succeed())

By("By updating the clusterResourcePlacement spec")
revisionLimit := int32(3)
createdCRP.Spec.RevisionHistoryLimit = &revisionLimit
Expect(k8sClient.Update(ctx, createdCRP)).Should(Succeed())

By("By checking placement controller queue")
Eventually(func() bool {
return fakePlacementController.Key() == testCRPName
}, eventuallyTimeout, interval).Should(BeTrue(), "placementController should receive the CRP name")
})

It("Updating the status and it should ignore the event", func() {
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: testCRPName}, createdCRP)).Should(Succeed())

By("By updating the clusterResourcePlacement status")
newCondition := metav1.Condition{
Type: string(fleetv1beta1.ClusterResourcePlacementAppliedConditionType),
Status: metav1.ConditionTrue,
Reason: "applied",
ObservedGeneration: createdCRP.GetGeneration(),
}
createdCRP.SetConditions(newCondition)
Expect(k8sClient.Status().Update(ctx, createdCRP)).Should(Succeed())

By("By checking placement controller queue")
Consistently(func() bool {
return fakePlacementController.Key() == ""
}, consistentlyDuration, interval).Should(BeTrue(), "watcher should ignore the update status event and not enqueue the request into the placementController queue")
})
})

Context("When deleting clusterResourcePlacement", func() {
BeforeEach(func() {
By("By resetting the placement queue")
fakePlacementController.ResetQueue()
})

It("Should enqueue the event", func() {
By("By deleting crp")
createdCRP = clusterResourcePlacementForTest()
Expect(k8sClient.Delete(ctx, createdCRP)).Should(Succeed())

By("By checking crp")
Eventually(func() bool {
return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testCRPName}, createdCRP))
}, eventuallyTimeout, interval).Should(BeTrue(), "crp should be deleted")

By("By checking placement controller queue")
Eventually(func() bool {
return fakePlacementController.Key() == testCRPName
}, eventuallyTimeout, interval).Should(BeTrue(), "placementController should receive the CRP name")
})
})
})
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"

Expand Down Expand Up @@ -40,9 +41,9 @@ func policySnapshot() *fleetv1beta1.ClusterSchedulingPolicySnapshot {

var _ = Describe("Test clusterSchedulingPolicySnapshot Controller", func() {
const (
timeout = time.Second * 10
duration = time.Second * 10
interval = time.Millisecond * 250
eventuallyTimeout = time.Second * 10
consistentlyDuration = time.Second * 10
interval = time.Millisecond * 250
)

var (
Expand All @@ -59,65 +60,94 @@ var _ = Describe("Test clusterSchedulingPolicySnapshot Controller", func() {
Context("When creating new clusterSchedulingPolicySnapshot", func() {
AfterEach(func() {
By("By deleting snapshot")
createdSnapshot := policySnapshot()
createdSnapshot = policySnapshot()
Expect(k8sClient.Delete(ctx, createdSnapshot)).Should(Succeed())

By("By checking snapshot")
Eventually(func() bool {
return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testSnapshotName}, createdSnapshot))
}, duration, interval).Should(BeTrue(), "snapshot should be deleted")
}, eventuallyTimeout, interval).Should(BeTrue(), "snapshot should be deleted")
})

It("Should ignore the event", func() {
By("By checking placement controller queue")
Consistently(func() bool {
return fakePlacementController.Key() == ""
}, duration, interval).Should(BeTrue(), "controller should ignore the create event and not enqueue the request into the placementController queue")
}, consistentlyDuration, interval).Should(BeTrue(), "controller should ignore the create event and not enqueue the request into the placementController queue")

})
})

Context("When updating clusterSchedulingPolicySnapshot", func() {
BeforeEach(func() {
By("By resetting the placement queue")
fakePlacementController.ResetQueue()
})

AfterEach(func() {
By("By deleting snapshot")
createdSnapshot := policySnapshot()
createdSnapshot = policySnapshot()
Expect(k8sClient.Delete(ctx, createdSnapshot)).Should(Succeed())

By("By checking snapshot")
Eventually(func() bool {
return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testSnapshotName}, createdSnapshot))
}, duration, interval).Should(BeTrue(), "snapshot should be deleted")
}, eventuallyTimeout, interval).Should(BeTrue(), "snapshot should be deleted")
})

It("Should enqueue the event", func() {
It("Updating the spec and should enqueue the event", func() {
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: testSnapshotName}, createdSnapshot)).Should(Succeed())

By("By updating the clusterSchedulingPolicySnapshot")
By("By updating the clusterSchedulingPolicySnapshot spec")
createdSnapshot.Spec.PolicyHash = []byte("modified-hash")
Expect(k8sClient.Update(ctx, createdSnapshot)).Should(Succeed())

By("By checking placement controller queue")
Eventually(func() bool {
return fakePlacementController.Key() == testCRPName
}, timeout, interval).Should(BeTrue(), "placementController should receive the CRP name")
}, eventuallyTimeout, interval).Should(BeTrue(), "placementController should receive the CRP name")
})

It("Updating the status and should enqueue the event", func() {
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: testSnapshotName}, createdSnapshot)).Should(Succeed())

By("By updating the clusterSchedulingPolicySnapshot status")
newCondition := metav1.Condition{
Type: string(fleetv1beta1.PolicySnapshotScheduled),
Status: metav1.ConditionTrue,
Reason: "scheduled",
ObservedGeneration: createdSnapshot.GetGeneration(),
}
meta.SetStatusCondition(&createdSnapshot.Status.Conditions, newCondition)
Expect(k8sClient.Status().Update(ctx, createdSnapshot)).Should(Succeed())

By("By checking placement controller queue")
Eventually(func() bool {
return fakePlacementController.Key() == testCRPName
}, eventuallyTimeout, interval).Should(BeTrue(), "placementController should receive the CRP name")
})
})

Context("When deleting clusterSchedulingPolicySnapshot", func() {
BeforeEach(func() {
By("By resetting the placement queue")
fakePlacementController.ResetQueue()
})

It("Should ignore the event", func() {
By("By deleting snapshot")
createdSnapshot := policySnapshot()
createdSnapshot = policySnapshot()
Expect(k8sClient.Delete(ctx, createdSnapshot)).Should(Succeed())

By("By checking snapshot")
Eventually(func() bool {
return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: testSnapshotName}, createdSnapshot))
}, duration, interval).Should(BeTrue(), "snapshot should be deleted")
}, eventuallyTimeout, interval).Should(BeTrue(), "snapshot should be deleted")

By("By checking placement controller queue")
Consistently(func() bool {
return fakePlacementController.Key() == ""
}, duration, interval).Should(BeTrue(), "controller should ignore the delete event and not enqueue the request into the placementController queue")
}, consistentlyDuration, interval).Should(BeTrue(), "controller should ignore the delete event and not enqueue the request into the placementController queue")
})
})
})

0 comments on commit 091ccc8

Please sign in to comment.