From 9e6b2ce15d61c7501ac0d42319cdede20662df45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 6 Sep 2024 14:33:13 +0200 Subject: [PATCH 1/6] feat: e2e test to demonstrate namespace deletion problem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../controller-namespace-deletion/pom.xml | 92 +++++++++++++++++++ ...rollerNamespaceDeletionCustomResource.java | 7 ++ .../ControllerNamespaceDeletionOperator.java | 19 ++++ ...ControllerNamespaceDeletionReconciler.java | 33 +++++++ .../ControllerNamespaceDeletionSpec.java | 6 ++ .../ControllerNamespaceDeletionStatus.java | 6 ++ .../src/main/resources/log4j2.xml | 13 +++ .../src/test/resources/log4j2.xml | 13 +++ sample-operators/pom.xml | 1 + 9 files changed, 190 insertions(+) create mode 100644 sample-operators/controller-namespace-deletion/pom.xml create mode 100644 sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java create mode 100644 sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java create mode 100644 sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java create mode 100644 sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java create mode 100644 sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java create mode 100644 sample-operators/controller-namespace-deletion/src/main/resources/log4j2.xml create mode 100644 sample-operators/controller-namespace-deletion/src/test/resources/log4j2.xml diff --git a/sample-operators/controller-namespace-deletion/pom.xml b/sample-operators/controller-namespace-deletion/pom.xml new file mode 100644 index 0000000000..7e7467cd8d --- /dev/null +++ b/sample-operators/controller-namespace-deletion/pom.xml @@ -0,0 +1,92 @@ + + + 4.0.0 + + + io.javaoperatorsdk + sample-operators + 5.0.0-SNAPSHOT + + + sample-controller-namespace-deletion + jar + Operator SDK - Samples - Controller Namespace Deletion + Deleting namespace with controller and custom resources + + + + + io.javaoperatorsdk + operator-framework-bom + ${project.version} + pom + import + + + + + + + io.javaoperatorsdk + operator-framework + + + io.fabric8 + crd-generator-apt + provided + + + org.apache.logging.log4j + log4j-slf4j2-impl + compile + + + org.apache.logging.log4j + log4j-core + compile + + + org.takes + takes + 1.24.4 + + + org.awaitility + awaitility + compile + + + io.javaoperatorsdk + operator-framework-junit-5 + test + + + org.junit.jupiter + junit-jupiter-params + test + + + + + + com.google.cloud.tools + jib-maven-plugin + ${jib-maven-plugin.version} + + + gcr.io/distroless/java17-debian11 + + + leader-election-operator + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.12.1 + + + + + diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java new file mode 100644 index 0000000000..caede8a760 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java @@ -0,0 +1,7 @@ +package io.javaoperatorsdk.operator.sample; + +import io.fabric8.kubernetes.client.CustomResource; + +public class ControllerNamespaceDeletionCustomResource extends CustomResource { + +} diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java new file mode 100644 index 0000000000..76883e96b8 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java @@ -0,0 +1,19 @@ +package io.javaoperatorsdk.operator.sample; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.javaoperatorsdk.operator.Operator; +import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration; + +public class ControllerNamespaceDeletionOperator { + + private static final Logger log = LoggerFactory.getLogger(ControllerNamespaceDeletionOperator.class); + + public static void main(String[] args) { + Operator operator = new Operator(); + + operator.register(new ControllerNamespaceDeletionReconciler()); + operator.start(); + } +} diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java new file mode 100644 index 0000000000..f8cd861408 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java @@ -0,0 +1,33 @@ +package io.javaoperatorsdk.operator.sample; + +import io.javaoperatorsdk.operator.api.reconciler.Cleaner; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.DeleteControl; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; + +import java.time.Duration; + +public class ControllerNamespaceDeletionReconciler implements Reconciler, + Cleaner { + + @Override + public UpdateControl reconcile(ControllerNamespaceDeletionCustomResource resource, + Context context) { + + + return null; + } + + + @Override + public DeleteControl cleanup(ControllerNamespaceDeletionCustomResource resource, + Context context) { + try { + Thread.sleep(Duration.ofSeconds(10).toMillis()); + return DeleteControl.defaultDelete(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java new file mode 100644 index 0000000000..25bd7a1f1c --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java @@ -0,0 +1,6 @@ +package io.javaoperatorsdk.operator.sample; + +import io.fabric8.kubernetes.client.CustomResource; + +public class ControllerNamespaceDeletionSpec { +} diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java new file mode 100644 index 0000000000..11b0de4219 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java @@ -0,0 +1,6 @@ +package io.javaoperatorsdk.operator.sample; + +import io.fabric8.kubernetes.client.CustomResource; + +public class ControllerNamespaceDeletionStatus { +} diff --git a/sample-operators/controller-namespace-deletion/src/main/resources/log4j2.xml b/sample-operators/controller-namespace-deletion/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..0ec69bf713 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/main/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/sample-operators/controller-namespace-deletion/src/test/resources/log4j2.xml b/sample-operators/controller-namespace-deletion/src/test/resources/log4j2.xml new file mode 100644 index 0000000000..2b7fdd3479 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/test/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/sample-operators/pom.xml b/sample-operators/pom.xml index 6d09f9a3ad..478508f9d5 100644 --- a/sample-operators/pom.xml +++ b/sample-operators/pom.xml @@ -17,5 +17,6 @@ webpage mysql-schema leader-election + controller-namespace-deletion From 946c4beefd42f12e1a2eae9d7f03436969ee426d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 6 Sep 2024 14:49:20 +0200 Subject: [PATCH 2/6] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- ...rollerNamespaceDeletionCustomResource.java | 7 +- .../ControllerNamespaceDeletionOperator.java | 4 +- ...ControllerNamespaceDeletionReconciler.java | 71 +++++++++++++------ .../ControllerNamespaceDeletionSpec.java | 11 ++- .../ControllerNamespaceDeletionStatus.java | 11 ++- 5 files changed, 76 insertions(+), 28 deletions(-) diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java index caede8a760..8aa9e5fc69 100644 --- a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java @@ -1,7 +1,12 @@ package io.javaoperatorsdk.operator.sample; import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.Version; -public class ControllerNamespaceDeletionCustomResource extends CustomResource { +@Group("namespacedeletion.io") +@Version("v1") +public class ControllerNamespaceDeletionCustomResource + extends CustomResource { } diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java index 76883e96b8..14b528be57 100644 --- a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java @@ -4,11 +4,11 @@ import org.slf4j.LoggerFactory; import io.javaoperatorsdk.operator.Operator; -import io.javaoperatorsdk.operator.api.config.LeaderElectionConfiguration; public class ControllerNamespaceDeletionOperator { - private static final Logger log = LoggerFactory.getLogger(ControllerNamespaceDeletionOperator.class); + private static final Logger log = + LoggerFactory.getLogger(ControllerNamespaceDeletionOperator.class); public static void main(String[] args) { Operator operator = new Operator(); diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java index f8cd861408..a504d5c9b8 100644 --- a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java @@ -1,33 +1,58 @@ package io.javaoperatorsdk.operator.sample; +import java.time.Duration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; import io.javaoperatorsdk.operator.api.reconciler.Cleaner; import io.javaoperatorsdk.operator.api.reconciler.Context; import io.javaoperatorsdk.operator.api.reconciler.DeleteControl; import io.javaoperatorsdk.operator.api.reconciler.Reconciler; import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; -import java.time.Duration; - -public class ControllerNamespaceDeletionReconciler implements Reconciler, - Cleaner { - - @Override - public UpdateControl reconcile(ControllerNamespaceDeletionCustomResource resource, - Context context) { - - - return null; - } - - - @Override - public DeleteControl cleanup(ControllerNamespaceDeletionCustomResource resource, - Context context) { - try { - Thread.sleep(Duration.ofSeconds(10).toMillis()); - return DeleteControl.defaultDelete(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } +public class ControllerNamespaceDeletionReconciler + implements Reconciler, + Cleaner { + + private static final Logger log = + LoggerFactory.getLogger(ControllerNamespaceDeletionReconciler.class); + + public static final Duration CLEANUP_DELAY = Duration.ofSeconds(10); + + @Override + public UpdateControl reconcile( + ControllerNamespaceDeletionCustomResource resource, + Context context) { + log.info("Reconciling: {} in namespace: {}", resource.getMetadata().getName(), + resource.getMetadata().getNamespace()); + + var response = createResponseResource(resource); + response.getStatus().setValue(resource.getSpec().getValue()); + + return UpdateControl.patchStatus(response); + } + + private ControllerNamespaceDeletionCustomResource createResponseResource( + ControllerNamespaceDeletionCustomResource resource) { + var res = new ControllerNamespaceDeletionCustomResource(); + res.setMetadata(new ObjectMetaBuilder() + .withName(resource.getMetadata().getName()) + .withNamespace(resource.getMetadata().getNamespace()) + .build()); + res.setStatus(new ControllerNamespaceDeletionStatus()); + return res; + } + + @Override + public DeleteControl cleanup(ControllerNamespaceDeletionCustomResource resource, + Context context) { + try { + Thread.sleep(CLEANUP_DELAY.toMillis()); + return DeleteControl.defaultDelete(); + } catch (InterruptedException e) { + throw new RuntimeException(e); } + } } diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java index 25bd7a1f1c..dc5092e7e5 100644 --- a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionSpec.java @@ -1,6 +1,15 @@ package io.javaoperatorsdk.operator.sample; -import io.fabric8.kubernetes.client.CustomResource; public class ControllerNamespaceDeletionSpec { + + private String value; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } } diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java index 11b0de4219..732fa7d626 100644 --- a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionStatus.java @@ -1,6 +1,15 @@ package io.javaoperatorsdk.operator.sample; -import io.fabric8.kubernetes.client.CustomResource; public class ControllerNamespaceDeletionStatus { + + private String value; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } } From 579e8c259d143e7b8bbdf5e226adf173af63535c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Mon, 16 Sep 2024 17:16:11 +0200 Subject: [PATCH 3/6] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../k8s/operator.yaml | 60 ++++++++ .../controller-namespace-deletion/pom.xml | 2 +- ...rollerNamespaceDeletionCustomResource.java | 4 +- .../ControllerNamespaceDeletionOperator.java | 31 +++- .../ControllerNamespaceDeletionE2E.java | 140 ++++++++++++++++++ 5 files changed, 233 insertions(+), 4 deletions(-) create mode 100644 sample-operators/controller-namespace-deletion/k8s/operator.yaml create mode 100644 sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java diff --git a/sample-operators/controller-namespace-deletion/k8s/operator.yaml b/sample-operators/controller-namespace-deletion/k8s/operator.yaml new file mode 100644 index 0000000000..f42f46cd50 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/k8s/operator.yaml @@ -0,0 +1,60 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator + +--- +apiVersion: v1 +kind: Pod +metadata: + name: operator +spec: + serviceAccountName: operator + containers: + - name: operator + image: controller-namespace-deletion-operator + imagePullPolicy: Never + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + terminationGracePeriodSeconds: 30 + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: operator + finalizers: + - controller.deletion/finalizer +subjects: + - kind: ServiceAccount + name: operator +roleRef: + kind: Role + name: operator + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operator + finalizers: + - controller.deletion/finalizer +rules: + - apiGroups: + - "apiextensions.k8s.io" + resources: + - customresourcedefinitions + verbs: + - '*' + - apiGroups: + - "namespacedeletion.io" + resources: + - controllernamespacedeletioncustomresources + - controllernamespacedeletioncustomresources/status + verbs: + - '*' + diff --git a/sample-operators/controller-namespace-deletion/pom.xml b/sample-operators/controller-namespace-deletion/pom.xml index 7e7467cd8d..c82272294b 100644 --- a/sample-operators/controller-namespace-deletion/pom.xml +++ b/sample-operators/controller-namespace-deletion/pom.xml @@ -77,7 +77,7 @@ gcr.io/distroless/java17-debian11 - leader-election-operator + controller-namespace-deletion-operator diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java index 8aa9e5fc69..ae0f1034ee 100644 --- a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionCustomResource.java @@ -1,5 +1,6 @@ package io.javaoperatorsdk.operator.sample; +import io.fabric8.kubernetes.api.model.Namespaced; import io.fabric8.kubernetes.client.CustomResource; import io.fabric8.kubernetes.model.annotation.Group; import io.fabric8.kubernetes.model.annotation.Version; @@ -7,6 +8,7 @@ @Group("namespacedeletion.io") @Version("v1") public class ControllerNamespaceDeletionCustomResource - extends CustomResource { + extends CustomResource + implements Namespaced { } diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java index 14b528be57..9cf719fbc0 100644 --- a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java @@ -1,9 +1,15 @@ package io.javaoperatorsdk.operator.sample; +import java.time.LocalTime; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.fabric8.kubernetes.client.KubernetesClientBuilder; import io.javaoperatorsdk.operator.Operator; +import io.javaoperatorsdk.operator.api.config.ControllerConfigurationOverrider; + +import static java.time.temporal.ChronoUnit.SECONDS; public class ControllerNamespaceDeletionOperator { @@ -11,9 +17,30 @@ public class ControllerNamespaceDeletionOperator { LoggerFactory.getLogger(ControllerNamespaceDeletionOperator.class); public static void main(String[] args) { - Operator operator = new Operator(); - operator.register(new ControllerNamespaceDeletionReconciler()); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + log.info("Shutting down..."); + boolean allResourcesDeleted = waitUntilResourcesDeleted(); + log.info("All resources within timeout: {}", allResourcesDeleted); + })); + + Operator operator = new Operator(); + operator.register(new ControllerNamespaceDeletionReconciler(), + ControllerConfigurationOverrider::watchingOnlyCurrentNamespace); operator.start(); } + + private static boolean waitUntilResourcesDeleted() { + try (var client = new KubernetesClientBuilder().build()) { + var startTime = LocalTime.now(); + while (startTime.until(LocalTime.now(), SECONDS) < 30) { + var items = + client.resources(ControllerNamespaceDeletionCustomResource.class).list().getItems(); + if (items.isEmpty()) { + return true; + } + } + return false; + } + } } diff --git a/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java b/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java new file mode 100644 index 0000000000..11a6e0bc0c --- /dev/null +++ b/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java @@ -0,0 +1,140 @@ +package io.javaoperatorsdk.operator.sample; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.NamespaceBuilder; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.rbac.RoleBinding; +import io.fabric8.kubernetes.client.ConfigBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientBuilder; + +import static io.javaoperatorsdk.operator.junit.AbstractOperatorExtension.CRD_READY_WAIT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + + +class ControllerNamespaceDeletionE2E { + + private static final Logger log = LoggerFactory.getLogger(ControllerNamespaceDeletionE2E.class); + + public static final String TEST_RESOURCE_NAME = "test1"; + public static final String INITIAL_VALUE = "initial value"; + public static final String ROLE_ROLE_BINDING_FINALIZER = "controller.deletion/finalizer"; + public static final String RESOURCE_NAME = "operator"; + + String namespace; + KubernetesClient client; + + // not for local mode by design + // @EnabledIfSystemProperty(named = "test.deployment", matches = "remote") + @Test + void customResourceCleanedUpOnNamespaceDeletion() { + deployController(); + client.resource(testResource()).serverSideApply(); + + await().untilAsserted(() -> { + var res = client.resources(ControllerNamespaceDeletionCustomResource.class) + .inNamespace(namespace).withName(TEST_RESOURCE_NAME).get(); + assertThat(res.getStatus()).isNotNull(); + assertThat(res.getStatus().getValue()).isEqualTo(INITIAL_VALUE); + }); + + client.namespaces().withName(namespace).delete(); + + await().timeout(Duration.ofSeconds(20)).untilAsserted(() -> { + var ns = client.resources(ControllerNamespaceDeletionCustomResource.class) + .inNamespace(namespace).withName(TEST_RESOURCE_NAME).get(); + assertThat(ns).isNull(); + }); + + log.info("Removing finalizers from role and role bing"); + removeRoleAndRoleBindingFinalizers(); + + await().untilAsserted(() -> { + var ns = client.namespaces().withName(namespace).get(); + assertThat(ns).isNull(); + }); + } + + private void removeRoleAndRoleBindingFinalizers() { + var rolebinding = + client.rbac().roleBindings().inNamespace(namespace).withName(RESOURCE_NAME).get(); + rolebinding.getFinalizers().clear(); + client.resource(rolebinding).update(); + + var role = client.rbac().roles().inNamespace(namespace).withName(RESOURCE_NAME).get(); + role.getFinalizers().clear(); + client.resource(role).update(); + + } + + ControllerNamespaceDeletionCustomResource testResource() { + var cr = new ControllerNamespaceDeletionCustomResource(); + cr.setMetadata(new ObjectMetaBuilder() + .withName(TEST_RESOURCE_NAME) + .withNamespace(namespace) + .build()); + cr.setSpec(new ControllerNamespaceDeletionSpec()); + cr.getSpec().setValue(INITIAL_VALUE); + return cr; + } + + + @BeforeEach + void setup() { + namespace = "controller-namespace-" + UUID.randomUUID(); + client = new KubernetesClientBuilder().withConfig(new ConfigBuilder() + .withNamespace(namespace) + .build()).build(); + applyCRD(); + client.namespaces().resource(new NamespaceBuilder().withNewMetadata().withName(namespace) + .endMetadata().build()).create(); + } + + void deployController() { + try { + List resources = client.load(new FileInputStream("k8s/operator.yaml")).items(); + resources.forEach(hm -> { + hm.getMetadata().setNamespace(namespace); + if (hm.getKind().equalsIgnoreCase("rolebinding")) { + var crb = (RoleBinding) hm; + for (var subject : crb.getSubjects()) { + subject.setNamespace(namespace); + } + } + }); + client.resourceList(resources) + .inNamespace(namespace) + .createOrReplace(); + + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + } + + void applyCRD() { + String path = + "target/classes/META-INF/fabric8/controllernamespacedeletioncustomresources.namespacedeletion.io-v1.yml"; + try (InputStream is = new FileInputStream(path)) { + final var crd = client.load(is); + crd.serverSideApply(); + Thread.sleep(CRD_READY_WAIT); + log.debug("Applied CRD with name: {}", crd.get().get(0).getMetadata().getName()); + } catch (InterruptedException | IOException e) { + throw new RuntimeException(e); + } + } +} From a9292eda733029dc2b3892b27651cc185d49d1f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 17 Sep 2024 11:06:44 +0200 Subject: [PATCH 4/6] impl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../controller-namespace-deletion/k8s/operator.yaml | 2 ++ .../sample/ControllerNamespaceDeletionOperator.java | 7 +++++-- .../sample/ControllerNamespaceDeletionReconciler.java | 1 + .../operator/sample/ControllerNamespaceDeletionE2E.java | 5 ++++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/sample-operators/controller-namespace-deletion/k8s/operator.yaml b/sample-operators/controller-namespace-deletion/k8s/operator.yaml index f42f46cd50..bc9eeb84ed 100644 --- a/sample-operators/controller-namespace-deletion/k8s/operator.yaml +++ b/sample-operators/controller-namespace-deletion/k8s/operator.yaml @@ -2,6 +2,8 @@ apiVersion: v1 kind: ServiceAccount metadata: name: operator + finalizers: + - controller.deletion/finalizer --- apiVersion: v1 diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java index 9cf719fbc0..5364852467 100644 --- a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionOperator.java @@ -33,9 +33,12 @@ public static void main(String[] args) { private static boolean waitUntilResourcesDeleted() { try (var client = new KubernetesClientBuilder().build()) { var startTime = LocalTime.now(); - while (startTime.until(LocalTime.now(), SECONDS) < 30) { + while (startTime.until(LocalTime.now(), SECONDS) < 20) { var items = - client.resources(ControllerNamespaceDeletionCustomResource.class).list().getItems(); + client.resources(ControllerNamespaceDeletionCustomResource.class) + .inNamespace(client.getConfiguration().getNamespace()) + .list().getItems(); + log.info("Custom resource in namespace: {}", items); if (items.isEmpty()) { return true; } diff --git a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java index a504d5c9b8..7261f269b4 100644 --- a/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java +++ b/sample-operators/controller-namespace-deletion/src/main/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionReconciler.java @@ -48,6 +48,7 @@ private ControllerNamespaceDeletionCustomResource createResponseResource( @Override public DeleteControl cleanup(ControllerNamespaceDeletionCustomResource resource, Context context) { + log.info("Cleaning up resource"); try { Thread.sleep(CLEANUP_DELAY.toMillis()); return DeleteControl.defaultDelete(); diff --git a/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java b/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java index 11a6e0bc0c..b0be2722a6 100644 --- a/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java +++ b/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java @@ -63,7 +63,7 @@ void customResourceCleanedUpOnNamespaceDeletion() { log.info("Removing finalizers from role and role bing"); removeRoleAndRoleBindingFinalizers(); - await().untilAsserted(() -> { + await().timeout(Duration.ofSeconds(20)).untilAsserted(() -> { var ns = client.namespaces().withName(namespace).get(); assertThat(ns).isNull(); }); @@ -79,6 +79,9 @@ private void removeRoleAndRoleBindingFinalizers() { role.getFinalizers().clear(); client.resource(role).update(); + var sa = client.serviceAccounts().inNamespace(namespace).withName(RESOURCE_NAME).get(); + sa.getMetadata().getFinalizers().clear(); + client.resource(sa).update(); } ControllerNamespaceDeletionCustomResource testResource() { From 015448e98f8c7479a7b38a67aa9c829bbddc03d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 17 Sep 2024 11:22:42 +0200 Subject: [PATCH 5/6] docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- sample-operators/controller-namespace-deletion/README.md | 8 ++++++++ .../operator/sample/ControllerNamespaceDeletionE2E.java | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 sample-operators/controller-namespace-deletion/README.md diff --git a/sample-operators/controller-namespace-deletion/README.md b/sample-operators/controller-namespace-deletion/README.md new file mode 100644 index 0000000000..3ea02d1d36 --- /dev/null +++ b/sample-operators/controller-namespace-deletion/README.md @@ -0,0 +1,8 @@ +This sample demonstrates the workaround for problem when a namespace +is being deleted with a running controller, that watches resources +in its own namespace. If the pod or other underlying resources (role, +role binding, service account) are deleted before the cleanup of +the custom resource the namespace deletion is stuck. + +see also: https://github.com/operator-framework/java-operator-sdk/pull/2528 + diff --git a/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java b/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java index b0be2722a6..36c7f132ab 100644 --- a/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java +++ b/sample-operators/controller-namespace-deletion/src/test/java/io/javaoperatorsdk/operator/sample/ControllerNamespaceDeletionE2E.java @@ -10,6 +10,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +40,7 @@ class ControllerNamespaceDeletionE2E { KubernetesClient client; // not for local mode by design - // @EnabledIfSystemProperty(named = "test.deployment", matches = "remote") + @EnabledIfSystemProperty(named = "test.deployment", matches = "remote") @Test void customResourceCleanedUpOnNamespaceDeletion() { deployController(); @@ -60,7 +61,7 @@ void customResourceCleanedUpOnNamespaceDeletion() { assertThat(ns).isNull(); }); - log.info("Removing finalizers from role and role bing"); + log.info("Removing finalizers from role and role bing and service account"); removeRoleAndRoleBindingFinalizers(); await().timeout(Duration.ofSeconds(20)).untilAsserted(() -> { From 30772685bf660bdfcdfb2235d73e1491bd77902d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 18 Sep 2024 12:58:44 +0200 Subject: [PATCH 6/6] remove unnecessary compiler plugins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- sample-operators/controller-namespace-deletion/pom.xml | 5 ----- sample-operators/leader-election/pom.xml | 5 ----- sample-operators/mysql-schema/pom.xml | 5 ----- sample-operators/tomcat-operator/pom.xml | 5 ----- sample-operators/webpage/pom.xml | 5 ----- 5 files changed, 25 deletions(-) diff --git a/sample-operators/controller-namespace-deletion/pom.xml b/sample-operators/controller-namespace-deletion/pom.xml index c82272294b..4979eec0d1 100644 --- a/sample-operators/controller-namespace-deletion/pom.xml +++ b/sample-operators/controller-namespace-deletion/pom.xml @@ -81,11 +81,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.12.1 - diff --git a/sample-operators/leader-election/pom.xml b/sample-operators/leader-election/pom.xml index 894b96d988..d686634ad7 100644 --- a/sample-operators/leader-election/pom.xml +++ b/sample-operators/leader-election/pom.xml @@ -76,11 +76,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.12.1 - io.fabric8 diff --git a/sample-operators/mysql-schema/pom.xml b/sample-operators/mysql-schema/pom.xml index bc8207a92e..46d555966c 100644 --- a/sample-operators/mysql-schema/pom.xml +++ b/sample-operators/mysql-schema/pom.xml @@ -102,11 +102,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.12.1 - diff --git a/sample-operators/tomcat-operator/pom.xml b/sample-operators/tomcat-operator/pom.xml index 7af9893609..d22260c614 100644 --- a/sample-operators/tomcat-operator/pom.xml +++ b/sample-operators/tomcat-operator/pom.xml @@ -104,11 +104,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.12.1 - diff --git a/sample-operators/webpage/pom.xml b/sample-operators/webpage/pom.xml index 9982da49c4..34b6846b94 100644 --- a/sample-operators/webpage/pom.xml +++ b/sample-operators/webpage/pom.xml @@ -75,11 +75,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.12.1 -