Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Proof of upgrade failure #1360

Closed
wants to merge 8 commits into from
Closed

Proof of upgrade failure #1360

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2441f7c
Adds e2e test to ensure injector correctly injects daprd after
JoshVanL Oct 19, 2023
33e89e0
Linting
JoshVanL Oct 19, 2023
7cfbb33
More linting
JoshVanL Oct 19, 2023
ae861c5
Set appID on sleep deployment, and return true on found container
JoshVanL Oct 19, 2023
315dc79
Linting
JoshVanL Oct 19, 2023
fa3a0e3
Merge branch 'master' into proof-of-upgrade-failure
pravinpushkar Oct 31, 2023
6582be5
Merge branch 'master' into proof-of-upgrade-failure
mukundansundar Nov 29, 2023
f4d71c1
Merge branch 'master' into proof-of-upgrade-failure
mukundansundar Jan 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 120 additions & 19 deletions tests/e2e/common/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package common

import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
Expand All @@ -25,17 +26,17 @@ import (

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
core_v1 "k8s.io/api/core/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/dapr/cli/pkg/kubernetes"
"github.com/dapr/cli/tests/e2e/spawn"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
k8s "k8s.io/client-go/kubernetes"

"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"

"github.com/dapr/cli/pkg/kubernetes"
"github.com/dapr/cli/tests/e2e/spawn"
)

type Resource int
Expand Down Expand Up @@ -200,6 +201,7 @@ func GetTestsOnInstall(details VersionDetails, opts TestOptions) []TestCase {
{"apply and check httpendpoints exist " + details.RuntimeVersion, HTTPEndpointsTestOnInstallUpgrade(opts)},
{"check mtls " + details.RuntimeVersion, MTLSTestOnInstallUpgrade(opts)},
{"status check " + details.RuntimeVersion, StatusTestOnInstallUpgrade(details, opts)},
{"injector injects sidecar " + details.RuntimeVersion, SidecarInjects()},
}
}

Expand Down Expand Up @@ -417,7 +419,7 @@ func ClusterRoleBindingsTest(details VersionDetails, opts TestOptions) func(t *t
list, err := k8sClient.
RbacV1().
ClusterRoleBindings().
List(ctx, v1.ListOptions{
List(ctx, metav1.ListOptions{
Limit: 100,
Continue: listContinue,
})
Expand Down Expand Up @@ -454,7 +456,7 @@ func ClusterRolesTest(details VersionDetails, opts TestOptions) func(t *testing.

var listContinue string
for {
list, err := k8sClient.RbacV1().ClusterRoles().List(ctx, v1.ListOptions{
list, err := k8sClient.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{
Limit: 100,
Continue: listContinue,
})
Expand Down Expand Up @@ -497,7 +499,7 @@ func CRDTest(details VersionDetails, opts TestOptions) func(t *testing.T) {
list, err := apiextensionsClientSet.
ApiextensionsV1().
CustomResourceDefinitions().
List(ctx, v1.ListOptions{
List(ctx, metav1.ListOptions{
Limit: 100,
Continue: listContinue,
})
Expand Down Expand Up @@ -695,6 +697,105 @@ func CheckMTLSStatus(details VersionDetails, opts TestOptions, shouldWarningExis
}
}

func SidecarInjects() func(t *testing.T) {
return func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 360*time.Second)
t.Cleanup(cancel)

client, err := getClient()
require.NoError(t, err)

deploy, err := client.AppsV1().Deployments(DaprTestNamespace).Create(ctx, &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "test-sidecar-injected-",
Namespace: DaprTestNamespace,
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "sleep"}},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{"app": "sleep"},
Annotations: map[string]string{
"dapr.io/enabled": "true",
"dapr.io/app-id": "sleep",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "sleep",
Image: "alpine:3.11",
Args: []string{"sleep", "infinity"},
},
},
},
},
},
}, metav1.CreateOptions{})
require.NoError(t, err)

t.Cleanup(func() {
cctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
assert.NoError(t,
client.AppsV1().Deployments(DaprTestNamespace).Delete(cctx, deploy.Name, metav1.DeleteOptions{}),
)
})

perr := wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (bool, error) {
deploy, err = client.AppsV1().Deployments(DaprTestNamespace).Get(ctx, deploy.Name, metav1.GetOptions{})
if err != nil {
return false, err
}

if deploy.Status.ReadyReplicas != 1 {
return false, nil
}

replicas, err := client.AppsV1().ReplicaSets(DaprTestNamespace).List(ctx, metav1.ListOptions{})
if err != nil {
return false, err
}

var replica string
for _, r := range replicas.Items {
for _, owner := range r.OwnerReferences {
if owner.Kind == "Deployment" && owner.Name == deploy.Name {
replica = r.Name
break
}
}
if len(replica) > 0 {
break
}
}

if len(replica) == 0 {
return false, nil
}

pods, err := client.CoreV1().Pods(DaprTestNamespace).List(ctx, metav1.ListOptions{})
if err != nil {
return false, err
}

for _, pod := range pods.Items {
for _, owner := range pod.OwnerReferences {
if owner.Kind == "ReplicaSet" && owner.Name == replica {
if len(pod.Spec.Containers) != 2 || pod.Spec.Containers[1].Name != "daprd" {
return false, errors.New("expected injected daprd container")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @JoshVanL, the upgrade tests are failing with this error, could you please check?

}
return true, nil
}
}
}

return false, errors.New("failed to find injected daprd container")
})
require.NoError(t, perr)
}
}

// Unexported functions.

func (v VersionDetails) constructFoundMap(res Resource) map[string]bool {
Expand Down Expand Up @@ -998,7 +1099,7 @@ func validateThirdpartyPodsOnInit(t *testing.T) {
defer cancel()
k8sClient, err := getClient()
require.NoError(t, err)
list, err := k8sClient.CoreV1().Pods(thirdPartyDevNamespace).List(ctxt, v1.ListOptions{
list, err := k8sClient.CoreV1().Pods(thirdPartyDevNamespace).List(ctxt, metav1.ListOptions{
Limit: 100,
})
require.NoError(t, err)
Expand All @@ -1013,7 +1114,7 @@ func validateThirdpartyPodsOnInit(t *testing.T) {
for _, pod := range list.Items {
t.Log(pod.ObjectMeta.Name)
for component, prefix := range prefixes {
if pod.Status.Phase != core_v1.PodRunning {
if pod.Status.Phase != corev1.PodRunning {
continue
}
if !pod.Status.ContainerStatuses[0].Ready {
Expand All @@ -1033,7 +1134,7 @@ func validatePodsOnInstallUpgrade(t *testing.T, details VersionDetails) {
defer cancel()
k8sClient, err := getClient()
require.NoError(t, err)
list, err := k8sClient.CoreV1().Pods(DaprTestNamespace).List(ctxt, v1.ListOptions{
list, err := k8sClient.CoreV1().Pods(DaprTestNamespace).List(ctxt, metav1.ListOptions{
Limit: 100,
})
require.NoError(t, err)
Expand Down Expand Up @@ -1065,7 +1166,7 @@ func validatePodsOnInstallUpgrade(t *testing.T, details VersionDetails) {
for _, pod := range list.Items {
t.Log(pod.ObjectMeta.Name)
for component, prefix := range prefixes {
if pod.Status.Phase != core_v1.PodRunning {
if pod.Status.Phase != corev1.PodRunning {
continue
}
if !pod.Status.ContainerStatuses[0].Ready {
Expand Down Expand Up @@ -1107,7 +1208,7 @@ func waitPodDeletionDev(t *testing.T, done, podsDeleted chan struct{}) {
defer cancel()
k8sClient, err := getClient()
require.NoError(t, err, "error getting k8s client for pods check")
list, err := k8sClient.CoreV1().Pods(thirdPartyDevNamespace).List(ctxt, v1.ListOptions{
list, err := k8sClient.CoreV1().Pods(thirdPartyDevNamespace).List(ctxt, metav1.ListOptions{
Limit: 100,
})
require.NoError(t, err)
Expand All @@ -1122,7 +1223,7 @@ func waitPodDeletionDev(t *testing.T, done, podsDeleted chan struct{}) {
for _, pod := range list.Items {
t.Log(pod.ObjectMeta.Name)
for component, prefix := range prefixes {
if pod.Status.Phase != core_v1.PodRunning {
if pod.Status.Phase != corev1.PodRunning {
continue
}
if !pod.Status.ContainerStatuses[0].Ready {
Expand Down Expand Up @@ -1153,7 +1254,7 @@ func waitPodDeletion(t *testing.T, done, podsDeleted chan struct{}) {
defer cancel()
k8sClient, err := getClient()
require.NoError(t, err, "error getting k8s client for pods check")
list, err := k8sClient.CoreV1().Pods(DaprTestNamespace).List(ctxt, v1.ListOptions{
list, err := k8sClient.CoreV1().Pods(DaprTestNamespace).List(ctxt, metav1.ListOptions{
Limit: 100,
})
require.NoError(t, err, "error getting pods list from k8s")
Expand All @@ -1177,14 +1278,14 @@ func waitAllPodsRunning(t *testing.T, namespace string, haEnabled bool, done, po
defer cancel()
k8sClient, err := getClient()
require.NoError(t, err, "error getting k8s client for pods check")
list, err := k8sClient.CoreV1().Pods(namespace).List(ctxt, v1.ListOptions{
list, err := k8sClient.CoreV1().Pods(namespace).List(ctxt, metav1.ListOptions{
Limit: 100,
})
require.NoError(t, err, "error getting pods list from k8s")
countOfReadyPods := 0
for _, item := range list.Items {
// Check pods running, and containers ready.
if item.Status.Phase == core_v1.PodRunning && len(item.Status.ContainerStatuses) != 0 {
if item.Status.Phase == corev1.PodRunning && len(item.Status.ContainerStatuses) != 0 {
size := len(item.Status.ContainerStatuses)
for _, status := range item.Status.ContainerStatuses {
if status.Ready {
Expand Down
1 change: 1 addition & 0 deletions tests/e2e/upgrade/upgrade_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ func getTestsOnUpgrade(p upgradePath, installOpts, upgradeOpts common.TestOption
{Name: "previously applied http endpoints exist " + details.RuntimeVersion, Callable: common.HTTPEndpointsTestOnInstallUpgrade(upgradeOpts)},
{Name: "check mtls " + details.RuntimeVersion, Callable: common.MTLSTestOnInstallUpgrade(upgradeOpts)},
{Name: "status check " + details.RuntimeVersion, Callable: common.StatusTestOnInstallUpgrade(details, upgradeOpts)},
{Name: "injector check " + details.RuntimeVersion, Callable: common.SidecarInjects()},
}...)

// uninstall.
Expand Down
Loading