From 49985f55de64a8d5ce917118de4852de3d086ab7 Mon Sep 17 00:00:00 2001
From: Shrinjay <shrinjaymukherjee@gmail.com>
Date: Tue, 24 Jan 2023 16:14:26 -0700
Subject: [PATCH 01/27] add remote cluster functionality

---
 .../services/external/__init__.py             |  0
 .../services/external/k8s_client.py           |  7 ++
 .../services/external/k8s_client_factory.py   | 30 ++++++++
 .../services/processproxies/crd.py            |  9 ++-
 .../services/processproxies/k8s.py            | 76 ++++++++++++++++---
 .../kubernetes/scripts/launch_kubernetes.py   | 23 +++---
 .../scripts/launch_custom_resource.py         |  8 +-
 .../sparkoperator.k8s.io-v1beta2.yaml.j2      | 14 +---
 .../templates/deployment.yaml                 | 41 ++++++++++
 .../templates/eg-clusterrole.yaml             |  6 ++
 .../templates/kubeconfig-configmap.yaml       |  9 +++
 .../helm/enterprise-gateway/values.yaml       |  7 ++
 12 files changed, 188 insertions(+), 42 deletions(-)
 create mode 100644 enterprise_gateway/services/external/__init__.py
 create mode 100644 enterprise_gateway/services/external/k8s_client.py
 create mode 100644 enterprise_gateway/services/external/k8s_client_factory.py
 create mode 100644 etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml

diff --git a/enterprise_gateway/services/external/__init__.py b/enterprise_gateway/services/external/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/enterprise_gateway/services/external/k8s_client.py b/enterprise_gateway/services/external/k8s_client.py
new file mode 100644
index 000000000..2c6e157a9
--- /dev/null
+++ b/enterprise_gateway/services/external/k8s_client.py
@@ -0,0 +1,7 @@
+"""Instantiates a static global factory and a single atomic client"""
+import os
+from enterprise_gateway.services.external.k8s_client_factory import KubernetesClientFactory
+
+KUBERNETES_CLIENT_FACTORY = KubernetesClientFactory()
+kubernetes_client = KUBERNETES_CLIENT_FACTORY.get_kubernetes_client()
+
diff --git a/enterprise_gateway/services/external/k8s_client_factory.py b/enterprise_gateway/services/external/k8s_client_factory.py
new file mode 100644
index 000000000..c5f32968b
--- /dev/null
+++ b/enterprise_gateway/services/external/k8s_client_factory.py
@@ -0,0 +1,30 @@
+"""Contains factory to create kubernetes api client instances using a single confguration"""
+import os
+
+from kubernetes import client, config
+from traitlets.config import SingletonConfigurable
+
+
+class KubernetesClientFactory(SingletonConfigurable):
+    """Manages kubernetes client creation from environment variables"""
+    def __init__(self) -> None:
+        """Maintain a single configuration object and populate based on environment"""
+        super().__init__()
+
+    def get_kubernetes_client(self, get_remote_if_available=True) -> client.ApiClient:
+        """Get kubernetes api client with appropriate configuration"""
+        kubernetes_config: client.Configuration = client.Configuration()
+        if os.getenv("KUBERNETES_SERVICE_HOST"):
+            # Running inside cluster
+            if os.getenv('EG_USE_REMOTE_CLUSTER') and get_remote_if_available:
+                kubeconfig_path = os.environ.get('EG_REMOTE_CLUSTER_KUBECONFIG_PATH', '/etc/kube/config')
+                config.load_kube_config(client_configuration=kubernetes_config, config_file=kubeconfig_path)
+            else:
+                config.load_incluster_config(client_configuration=kubernetes_config)
+        else:
+            config.load_kube_config(client_configuration=kubernetes_config)
+
+        self.log.debug(
+            "Created kubernetes client for host {host}".format(host=kubernetes_config.host)
+        )
+        return client.ApiClient(kubernetes_config)
diff --git a/enterprise_gateway/services/processproxies/crd.py b/enterprise_gateway/services/processproxies/crd.py
index 392aa9489..ead851f42 100644
--- a/enterprise_gateway/services/processproxies/crd.py
+++ b/enterprise_gateway/services/processproxies/crd.py
@@ -5,9 +5,11 @@
 from __future__ import annotations
 
 from typing import Any
+import os
 
 from kubernetes import client
 
+from ..external.k8s_client import kubernetes_client
 from ..kernels.remotemanager import RemoteKernelManager
 from .k8s import KubernetesProcessProxy
 
@@ -37,6 +39,11 @@ async def launch_process(
         kwargs["env"]["KERNEL_CRD_VERSION"] = self.version
         kwargs["env"]["KERNEL_CRD_PLURAL"] = self.plural
 
+        use_remote_cluster = os.getenv("EG_USE_REMOTE_CLUSTER")
+        if use_remote_cluster:
+            kwargs["env"]["EG_USE_REMOTE_CLUSTER"] = 'true'
+            kwargs["env"]["EG_REMOTE_CLUSTER_KUBECONFIG_PATH"] = os.getenv("EG_REMOTE_CLUSTER_KUBECONFIG_PATH")
+
         await super().launch_process(kernel_cmd, **kwargs)
         return self
 
@@ -48,7 +55,7 @@ def delete_managed_object(self, termination_stati: list[str]) -> bool:
 
         Note: the caller is responsible for handling exceptions.
         """
-        delete_status = client.CustomObjectsApi().delete_namespaced_custom_object(
+        delete_status = client.CustomObjectsApi(api_client=kubernetes_client).delete_namespaced_custom_object(
             self.group,
             self.version,
             self.kernel_namespace,
diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index 0e6a7dff7..a3ffd69c3 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -7,11 +7,12 @@
 import logging
 import os
 import re
-from typing import Any
+from typing import Any, List
 
 import urllib3
-from kubernetes import client, config
+from kubernetes import client
 
+from ..external.k8s_client import kubernetes_client, KUBERNETES_CLIENT_FACTORY
 from ..kernels.remotemanager import RemoteKernelManager
 from ..sessions.kernelsessionmanager import KernelSessionManager
 from .container import ContainerProcessProxy
@@ -29,8 +30,6 @@
 share_gateway_namespace = bool(os.environ.get("EG_SHARED_NAMESPACE", "False").lower() == "true")
 kpt_dir = os.environ.get("EG_POD_TEMPLATE_DIR", "/tmp")
 
-config.load_incluster_config()
-
 
 class KubernetesProcessProxy(ContainerProcessProxy):
     """
@@ -81,7 +80,7 @@ def get_container_status(self, iteration: int | None) -> str | None:
         # is used for the assigned_ip.
         pod_status = None
         kernel_label_selector = "kernel_id=" + self.kernel_id + ",component=kernel"
-        ret = client.CoreV1Api().list_namespaced_pod(
+        ret = client.CoreV1Api(api_client=kubernetes_client).list_namespaced_pod(
             namespace=self.kernel_namespace, label_selector=kernel_label_selector
         )
         if ret and ret.items:
@@ -117,7 +116,7 @@ def delete_managed_object(self, termination_stati: list[str]) -> bool:
         # Deleting a Pod will return a v1.Pod if found and its status will be a PodStatus containing
         # a phase string property
         # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#podstatus-v1-core
-        v1_pod = client.CoreV1Api().delete_namespaced_pod(
+        v1_pod = client.CoreV1Api(api_client=kubernetes_client).delete_namespaced_pod(
             namespace=self.kernel_namespace, body=body, name=self.container_name
         )
         status = None
@@ -164,7 +163,7 @@ def terminate_container_resources(self) -> bool | None:
                 body = client.V1DeleteOptions(
                     grace_period_seconds=0, propagation_policy="Background"
                 )
-                v1_status = client.CoreV1Api().delete_namespace(
+                v1_status = client.CoreV1Api(api_client=kubernetes_client).delete_namespace(
                     name=self.kernel_namespace, body=body
                 )
                 status = None
@@ -281,10 +280,15 @@ def _create_kernel_namespace(self, service_account_name: str) -> str:
 
         # create the namespace
         try:
-            client.CoreV1Api().create_namespace(body=body)
+            client.CoreV1Api(api_client=kubernetes_client).create_namespace(body=body)
             self.delete_kernel_namespace = True
             self.log.info(f"Created kernel namespace: {namespace}")
 
+            if os.getenv('EG_USE_REMOTE_CLUSTER'):
+                # If remote cluster is being used, service account may not be present, create before role binding
+                self._create_service_account_if_not_exists(namespace=namespace,
+                                                           service_account_name=service_account_name)
+
             # Now create a RoleBinding for this namespace for the default ServiceAccount.  We'll reference
             # the ClusterRole, but that will only be applied for this namespace.  This prevents the need for
             # creating a role each time.
@@ -308,7 +312,7 @@ def _create_kernel_namespace(self, service_account_name: str) -> str:
                     body = client.V1DeleteOptions(
                         grace_period_seconds=0, propagation_policy="Background"
                     )
-                    client.CoreV1Api().delete_namespace(name=namespace, body=body)
+                    client.CoreV1Api(api_client=kubernetes_client).delete_namespace(name=namespace, body=body)
                     self.log.warning(f"Deleted kernel namespace: {namespace}")
                 else:
                     reason = f"Error occurred creating namespace '{namespace}': {err}"
@@ -316,6 +320,53 @@ def _create_kernel_namespace(self, service_account_name: str) -> str:
 
         return namespace
 
+    def _create_service_account_if_not_exists(self, namespace: str, service_account_name: str) -> None:
+        service_account_list_in_namespace: client.V1ServiceAccountList = client\
+            .CoreV1Api(api_client=kubernetes_client)\
+            .list_namespaced_service_account(namespace=namespace)
+
+        service_accounts_in_namespace: List[client.V1ServiceAccount] = service_account_list_in_namespace.items
+        service_account_names_in_namespace: List[str] = list(map(lambda svcaccount: svcaccount.metadata.name, service_accounts_in_namespace))
+
+        if service_account_name not in service_account_names_in_namespace:
+            service_account_metadata = {
+                "name": service_account_name
+            }
+            service_account_to_create: client.V1ServiceAccount = client.V1ServiceAccount(
+                kind="ServiceAccount",
+                metadata=service_account_metadata
+            )
+
+            client\
+                .CoreV1Api(api_client=kubernetes_client)\
+                .create_namespaced_service_account(namespace=namespace, body=service_account_to_create)
+
+            self.log.info(f"Created service account {service_account_name} in namespace {namespace}")
+
+    def _forward_role_to_remote(self) -> None:
+        remote_cluster_roles: client.V1ClusterRoleList = client\
+            .RbacAuthorizationV1Api(api_client=kubernetes_client)\
+            .list_cluster_role()
+        remote_cluster_role_names = list(map(lambda role: role.metadata.name, remote_cluster_roles.items))
+
+        if kernel_cluster_role not in remote_cluster_role_names:
+            incluster_client = KUBERNETES_CLIENT_FACTORY.get_kubernetes_client(get_remote_if_available=False)
+            incluster_role: client.V1Role = client \
+                .RbacAuthorizationV1Api(api_client=incluster_client) \
+                .read_cluster_role(kernel_cluster_role)
+
+            remote_cluster_role: client.V1Role = client.V1Role(
+                api_version=incluster_role.api_version,
+                kind=incluster_role.kind,
+                rules=incluster_role.rules,
+                metadata=incluster_role.metadata
+            )
+
+            remote_cluster_role.metadata.resource_version = None
+
+            client.RbacAuthorizationV1Api(api_client=kubernetes_client).create_cluster_role(body=remote_cluster_role)
+
+
     def _create_role_binding(self, namespace: str, service_account_name: str) -> None:
         # Creates RoleBinding instance for the given namespace.  The role used will be the ClusterRole named by
         # EG_KERNEL_CLUSTER_ROLE.
@@ -342,7 +393,12 @@ def _create_role_binding(self, namespace: str, service_account_name: str) -> Non
             subjects=[binding_subjects],
         )
 
-        client.RbacAuthorizationV1Api().create_namespaced_role_binding(
+        # If remote cluster is used, we need to create a cluster role on that cluster
+        # Forward the role defined via helm to the remote cluster
+        if os.environ.get('EG_USE_REMOTE_CLUSTER'):
+            self._forward_role_to_remote()
+
+        client.RbacAuthorizationV1Api(api_client=kubernetes_client).create_namespaced_role_binding(
             namespace=namespace, body=body
         )
         self.log.info(
diff --git a/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py b/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py
index 2582d66f2..16ae0288e 100644
--- a/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py
+++ b/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py
@@ -8,7 +8,8 @@
 import urllib3
 import yaml
 from jinja2 import Environment, FileSystemLoader, select_autoescape
-from kubernetes import client, config
+from kubernetes import client
+from enterprise_gateway.services.external.k8s_client import kubernetes_client
 from kubernetes.client.rest import ApiException
 
 urllib3.disable_warnings()
@@ -96,12 +97,6 @@ def launch_kubernetes_kernel(
     kernel_class_name,
 ):
     """Launches a containerized kernel as a kubernetes pod."""
-
-    if os.getenv("KUBERNETES_SERVICE_HOST"):
-        config.load_incluster_config()
-    else:
-        config.load_kube_config()
-
     # Capture keywords and their values.
     keywords = dict()
 
@@ -154,13 +149,13 @@ def launch_kubernetes_kernel(
                 pod_template = extend_pod_env(k8s_obj)
                 if pod_template_file is None:
                     try:
-                        pod_created = client.CoreV1Api(client.ApiClient()).create_namespaced_pod(
+                        pod_created = client.CoreV1Api(api_client=kubernetes_client).create_namespaced_pod(
                             body=k8s_obj, namespace=kernel_namespace
                         )
                     except ApiException as exc:
                         if _parse_k8s_exception(exc) == K8S_ALREADY_EXIST_REASON:
                             pod_created = (
-                                client.CoreV1Api(client.ApiClient())
+                                client.CoreV1Api(api_client=kubernetes_client)
                                 .list_namespaced_pod(
                                     namespace=kernel_namespace,
                                     label_selector=f"kernel_id={kernel_id}",
@@ -172,14 +167,14 @@ def launch_kubernetes_kernel(
                             raise exc
             elif k8s_obj["kind"] == "Secret":
                 if pod_template_file is None:
-                    client.CoreV1Api(client.ApiClient()).create_namespaced_secret(
+                    client.CoreV1Api(api_client=kubernetes_client).create_namespaced_secret(
                         body=k8s_obj, namespace=kernel_namespace
                     )
             elif k8s_obj["kind"] == "PersistentVolumeClaim":
                 if pod_template_file is None:
                     try:
                         client.CoreV1Api(
-                            client.ApiClient()
+                            api_client=kubernetes_client
                         ).create_namespaced_persistent_volume_claim(
                             body=k8s_obj, namespace=kernel_namespace
                         )
@@ -190,7 +185,7 @@ def launch_kubernetes_kernel(
                             raise exc
             elif k8s_obj["kind"] == "PersistentVolume":
                 if pod_template_file is None:
-                    client.CoreV1Api(client.ApiClient()).create_persistent_volume(body=k8s_obj)
+                    client.CoreV1Api(api_client=kubernetes_client).create_persistent_volume(body=k8s_obj)
             elif k8s_obj["kind"] == "Service":
                 if pod_template_file is None:
                     if pod_created is not None:
@@ -203,7 +198,7 @@ def launch_kubernetes_kernel(
                                 "uid": str(pod_created.metadata.uid),
                             }
                         ]
-                        client.CoreV1Api(client.ApiClient()).create_namespaced_service(
+                        client.CoreV1Api(api_client=kubernetes_client).create_namespaced_service(
                             body=k8s_obj, namespace=kernel_namespace
                         )
             elif k8s_obj["kind"] == "ConfigMap":
@@ -218,7 +213,7 @@ def launch_kubernetes_kernel(
                                 "uid": str(pod_created.metadata.uid),
                             }
                         ]
-                        client.CoreV1Api(client.ApiClient()).create_namespaced_config_map(
+                        client.CoreV1Api(api_client=kubernetes_client).create_namespaced_config_map(
                             body=k8s_obj, namespace=kernel_namespace
                         )
             else:
diff --git a/etc/kernel-launchers/operators/scripts/launch_custom_resource.py b/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
index 82a170bf9..d634ecf60 100644
--- a/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
+++ b/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
@@ -7,7 +7,8 @@
 import urllib3
 import yaml
 from jinja2 import Environment, FileSystemLoader, select_autoescape
-from kubernetes import client, config
+from kubernetes import client
+from enterprise_gateway.services.external.k8s_client import kubernetes_client
 
 urllib3.disable_warnings()
 
@@ -56,9 +57,6 @@ def extend_operator_env(op_def: dict, sub_spec: str) -> dict:
 def launch_custom_resource_kernel(
     kernel_id, port_range, response_addr, public_key, spark_context_init_mode
 ):
-    """Launch a custom resource kernel."""
-    config.load_incluster_config()
-
     keywords = dict()
 
     keywords["eg_port_range"] = port_range
@@ -87,7 +85,7 @@ def launch_custom_resource_kernel(
         extend_operator_env(custom_resource_object, "executor")
 
     try:
-        client.CustomObjectsApi().create_namespaced_custom_object(
+        client.CustomObjectsApi(api_client=kubernetes_client).create_namespaced_custom_object(
             group, version, kernel_namespace, plural, custom_resource_object
         )
     except client.exceptions.ApiException as ex:
diff --git a/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2 b/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
index c13c4687f..4af55fee1 100644
--- a/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
+++ b/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
@@ -2,6 +2,8 @@ apiVersion: "sparkoperator.k8s.io/v1beta2"
 kind: SparkApplication
 metadata:
   name: {{ kernel_resource_name }}
+  labels:
+    app.kubernetes.io/name: kernel
 spec:
   type: Python
   pythonVersion: "3"
@@ -20,12 +22,6 @@ spec:
     - "--public-key"
     - "{{ eg_public_key }}"
   driver:
-    env:
-# Add any custom envs here that aren't already configured for the kernel's environment
-# Note: For envs to flow to the pods, the webhook server must be enabled during deployment
-# e.g., helm install my-release spark-operator/spark-operator --namespace spark-operator --set webhook.enable=true
-#    - name: MY_DRIVER_ENV
-#      value: "my_driver_value"
     serviceAccount: "{{ kernel_service_account_name }}"
     labels:
       kernel_id: "{{ kernel_id }}"
@@ -35,12 +31,6 @@ spec:
     coreLimit: 1000m
     memory: 1g
   executor:
-    env:
-# Add any custom envs here that aren't already configured for the kernel's environment
-# Note: For envs to flow to the pods, the webhook server must be enabled during deployment
-# e.g., helm install my-release spark-operator/spark-operator --namespace spark-operator --set webhook.enable=true
-#    - name: MY_EXECUTOR_ENV
-#      value: "my_executor_value"
     labels:
       kernel_id: "{{ kernel_id }}"
       app: enterprise-gateway
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
index 4cc7371a6..6915a0d50 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
@@ -21,6 +21,9 @@ spec:
       gateway-selector: enterprise-gateway
   template:
     metadata:
+      annotations:
+        # Force redeploy when configmap updates
+        checksum/config: {{ include (print $.Template.BasePath "/kubeconfig-configmap.yaml") . | sha256sum }}
       labels:
         gateway-selector: enterprise-gateway
         app: enterprise-gateway
@@ -41,6 +44,25 @@ spec:
         volumeMounts:
         - name: image-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
+        {{- if .Values.remoteKubeconfig.enabled }}
+        - name: kubeconfig
+          readOnly: true
+          mountPath: {{ .Values.remoteKubeconfig.configPath }}
+        {{- end}}
+        volumes:
+        {{- if .Values.remoteKubeconfig.enabled }}
+        - name: kubeconfig
+          configMap: 
+            name: {{ .Release.Name }}-configmap
+        {{- end}}
+        env:
+        # Set environment variables to point k8s client to kubeconfig
+        {{- if .Values.remoteKubeconfig.enabled }}
+          - name: EG_USE_REMOTE_CLUSTER
+            value: "True"
+          - name: EG_REMOTE_CLUSTER_KUBECONFIG_PATH
+            value: {{ .Values.remoteKubeconfig.configPath }}config
+        {{- end }}
   {{- end }}
       containers:
       - name: enterprise-gateway
@@ -75,6 +97,15 @@ spec:
           value: {{ toJson .Values.kernel.allowedKernels | squote }}
         - name: EG_DEFAULT_KERNEL_NAME
           value: {{ .Values.kernel.defaultKernelName }}
+        # Set environment variables to point k8s client to kubeconfig
+        {{- if .Values.remoteKubeconfig.enabled }}
+        - name: EG_USE_REMOTE_CLUSTER
+          value: "True"
+        - name: EG_REMOTE_CLUSTER_KUBECONFIG_PATH
+          value: {{ .Values.remoteKubeconfig.configPath }}config
+        - name: EG_DEFAULT_KERNEL_SERVICE_ACCOUNT_NAME
+          value: kernel-svcaccount
+        {{- end }}
         # Optional authorization token passed in all requests
         {{- if .Values.authToken }}
         - name: EG_AUTH_TOKEN
@@ -120,10 +151,20 @@ spec:
         volumeMounts:
         - name: image-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
+        {{- if .Values.remoteKubeconfig.enabled }}
+        - name: kubeconfig
+          readOnly: true
+          mountPath: {{ .Values.remoteKubeconfig.configPath }}
+        {{- end}}
       volumes:
       - name: image-kernelspecs
         emptyDir:
           medium: Memory
+      {{- if .Values.remoteKubeconfig.enabled }}
+      - name: kubeconfig
+        configMap: 
+          name: {{ .Release.Name }}-configmap
+      {{- end}}
 
   {{- if .Values.deployment.tolerations }}
       tolerations:
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
index 11a0abac5..d38607dbb 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
@@ -23,6 +23,12 @@ rules:
   - apiGroups: ["sparkoperator.k8s.io"]
     resources: ["sparkapplications", "sparkapplications/status", "scheduledsparkapplications", "scheduledsparkapplications/status"]
     verbs: ["get", "watch", "list", "create", "delete"]
+  # If remoteKubeconfig used, need to be able to read and forward service account
+  {{- if .Values.remoteKubeconfig.enabled }}
+  - apiGroups: ["rbac.authorization.k8s.io"]
+    resources: ["clusterroles"]
+    verbs: ["get", "create"]
+  {{- end}}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
new file mode 100644
index 000000000..be9741102
--- /dev/null
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
@@ -0,0 +1,9 @@
+{{- if .Values.remoteKubeconfig.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-configmap
+data:
+  config: |
+{{ .Files.Get "kubeconfig" | indent 6 }}
+{{- end }}
\ No newline at end of file
diff --git a/etc/kubernetes/helm/enterprise-gateway/values.yaml b/etc/kubernetes/helm/enterprise-gateway/values.yaml
index 4fb3b6b29..86879c9e3 100644
--- a/etc/kubernetes/helm/enterprise-gateway/values.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/values.yaml
@@ -179,3 +179,10 @@ kip:
   #   requests:
   #     cpu: 1
   #     memory: 1Gi
+
+remoteKubeconfig:
+  # Set to enable to point k8s client to remote cluster
+  enabled: false
+  # Mount config at this path inside pod
+  configPath:
+

From d49150873d2b85695d5e3d9654f627cf4cb9b3b9 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Wed, 25 Jan 2023 12:57:37 -0700
Subject: [PATCH 02/27] Add context selection to helm and passthrough

---
 enterprise_gateway/services/external/k8s_client_factory.py    | 3 ++-
 .../helm/enterprise-gateway/templates/deployment.yaml         | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/enterprise_gateway/services/external/k8s_client_factory.py b/enterprise_gateway/services/external/k8s_client_factory.py
index c5f32968b..9b2568d0c 100644
--- a/enterprise_gateway/services/external/k8s_client_factory.py
+++ b/enterprise_gateway/services/external/k8s_client_factory.py
@@ -18,7 +18,8 @@ def get_kubernetes_client(self, get_remote_if_available=True) -> client.ApiClien
             # Running inside cluster
             if os.getenv('EG_USE_REMOTE_CLUSTER') and get_remote_if_available:
                 kubeconfig_path = os.environ.get('EG_REMOTE_CLUSTER_KUBECONFIG_PATH', '/etc/kube/config')
-                config.load_kube_config(client_configuration=kubernetes_config, config_file=kubeconfig_path)
+                context = os.environ.get('EG_REMOTE_CLUSTER_CONTEXT', None)
+                config.load_kube_config(client_configuration=kubernetes_config, config_file=kubeconfig_path, context=context)
             else:
                 config.load_incluster_config(client_configuration=kubernetes_config)
         else:
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
index 6915a0d50..9f95cfce6 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
@@ -105,6 +105,10 @@ spec:
           value: {{ .Values.remoteKubeconfig.configPath }}config
         - name: EG_DEFAULT_KERNEL_SERVICE_ACCOUNT_NAME
           value: kernel-svcaccount
+        {{- if .Values.remoteKubeconfig.context }}
+        - name: EG_REMOTE_CLUSTER_CONTEXT
+          value: {{ .Values.remoteKubeconfig.context }}
+        {{- end}}
         {{- end }}
         # Optional authorization token passed in all requests
         {{- if .Values.authToken }}

From 686ed39c1c82a63a24d49b82fcf211a6a5c297fd Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Wed, 25 Jan 2023 12:58:37 -0700
Subject: [PATCH 03/27] Remove label

---
 .../operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2      | 2 --
 1 file changed, 2 deletions(-)

diff --git a/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2 b/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
index 4af55fee1..371ae581f 100644
--- a/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
+++ b/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
@@ -2,8 +2,6 @@ apiVersion: "sparkoperator.k8s.io/v1beta2"
 kind: SparkApplication
 metadata:
   name: {{ kernel_resource_name }}
-  labels:
-    app.kubernetes.io/name: kernel
 spec:
   type: Python
   pythonVersion: "3"

From 7e9061126b9e1dd85ef0cebae268af72c6d246e9 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Wed, 25 Jan 2023 12:59:51 -0700
Subject: [PATCH 04/27] Put back comments

---
 .../scripts/sparkoperator.k8s.io-v1beta2.yaml.j2     | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2 b/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
index 371ae581f..e43649b92 100644
--- a/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
+++ b/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
@@ -20,6 +20,12 @@ spec:
     - "--public-key"
     - "{{ eg_public_key }}"
   driver:
+    env:
+# Add any custom envs here that aren't already configured for the kernel's environment
+# Note: For envs to flow to the pods, the webhook server must be enabled during deployment
+# e.g., helm install my-release spark-operator/spark-operator --namespace spark-operator --set webhook.enable=true
+#    - name: MY_DRIVER_ENV
+#      value: "my_driver_value"
     serviceAccount: "{{ kernel_service_account_name }}"
     labels:
       kernel_id: "{{ kernel_id }}"
@@ -29,6 +35,12 @@ spec:
     coreLimit: 1000m
     memory: 1g
   executor:
+    env:
+# Add any custom envs here that aren't already configured for the kernel's environment
+# Note: For envs to flow to the pods, the webhook server must be enabled during deployment
+# e.g., helm install my-release spark-operator/spark-operator --namespace spark-operator --set webhook.enable=true
+#    - name: MY_DRIVER_ENV
+#      value: "my_driver_value"
     labels:
       kernel_id: "{{ kernel_id }}"
       app: enterprise-gateway

From 2daf2ccbfe1a46beb316d8cb6197cd5a22d4ad39 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Wed, 25 Jan 2023 13:00:33 -0700
Subject: [PATCH 05/27] Fix names

---
 .../operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2 b/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
index e43649b92..c13c4687f 100644
--- a/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
+++ b/etc/kernel-launchers/operators/scripts/sparkoperator.k8s.io-v1beta2.yaml.j2
@@ -39,8 +39,8 @@ spec:
 # Add any custom envs here that aren't already configured for the kernel's environment
 # Note: For envs to flow to the pods, the webhook server must be enabled during deployment
 # e.g., helm install my-release spark-operator/spark-operator --namespace spark-operator --set webhook.enable=true
-#    - name: MY_DRIVER_ENV
-#      value: "my_driver_value"
+#    - name: MY_EXECUTOR_ENV
+#      value: "my_executor_value"
     labels:
       kernel_id: "{{ kernel_id }}"
       app: enterprise-gateway

From ff5aab561b9f916b1a2a115c044631d8f8c2207e Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Wed, 25 Jan 2023 16:11:40 -0700
Subject: [PATCH 06/27] Fix docstrings, generator, and various other

---
 .../services/external/k8s_client_factory.py   | 19 ++++++----
 .../services/processproxies/k8s.py            | 35 +++++++++++--------
 2 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/enterprise_gateway/services/external/k8s_client_factory.py b/enterprise_gateway/services/external/k8s_client_factory.py
index 9b2568d0c..b2d8c8469 100644
--- a/enterprise_gateway/services/external/k8s_client_factory.py
+++ b/enterprise_gateway/services/external/k8s_client_factory.py
@@ -11,14 +11,21 @@ def __init__(self) -> None:
         """Maintain a single configuration object and populate based on environment"""
         super().__init__()
 
-    def get_kubernetes_client(self, get_remote_if_available=True) -> client.ApiClient:
-        """Get kubernetes api client with appropriate configuration"""
+    def get_kubernetes_client(self, get_remote_client: bool = True) -> client.ApiClient:
+        """Get kubernetes api client with appropriate configuration
+
+        Args:
+            get_remote_client (bool): Return a client for the remote cluster if configured. Else, return incluster config. Defaults to True.
+
+        Returns:
+            ApiClient: Kubernetes API client for appropriate cluster
+        """
         kubernetes_config: client.Configuration = client.Configuration()
         if os.getenv("KUBERNETES_SERVICE_HOST"):
             # Running inside cluster
-            if os.getenv('EG_USE_REMOTE_CLUSTER') and get_remote_if_available:
-                kubeconfig_path = os.environ.get('EG_REMOTE_CLUSTER_KUBECONFIG_PATH', '/etc/kube/config')
-                context = os.environ.get('EG_REMOTE_CLUSTER_CONTEXT', None)
+            if os.getenv('EG_USE_REMOTE_CLUSTER') and get_remote_client:
+                kubeconfig_path = os.getenv('EG_REMOTE_CLUSTER_KUBECONFIG_PATH', '/etc/kube/config/')
+                context = os.getenv('EG_REMOTE_CLUSTER_CONTEXT', None)
                 config.load_kube_config(client_configuration=kubernetes_config, config_file=kubeconfig_path, context=context)
             else:
                 config.load_incluster_config(client_configuration=kubernetes_config)
@@ -26,6 +33,6 @@ def get_kubernetes_client(self, get_remote_if_available=True) -> client.ApiClien
             config.load_kube_config(client_configuration=kubernetes_config)
 
         self.log.debug(
-            "Created kubernetes client for host {host}".format(host=kubernetes_config.host)
+            f"Created kubernetes client for host {kubernetes_config.host}"
         )
         return client.ApiClient(kubernetes_config)
diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index a3ffd69c3..31914e6ed 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -321,12 +321,13 @@ def _create_kernel_namespace(self, service_account_name: str) -> str:
         return namespace
 
     def _create_service_account_if_not_exists(self, namespace: str, service_account_name: str) -> None:
-        service_account_list_in_namespace: client.V1ServiceAccountList = client\
-            .CoreV1Api(api_client=kubernetes_client)\
-            .list_namespaced_service_account(namespace=namespace)
+        """If service account doesn't exist in target cluster, create one. Occurs if a remote cluster is being used."""
+        service_account_list_in_namespace: client.V1ServiceAccountList = (client
+                                                                          .CoreV1Api(api_client=kubernetes_client)
+                                                                          .list_namespaced_service_account(namespace=namespace))
 
         service_accounts_in_namespace: List[client.V1ServiceAccount] = service_account_list_in_namespace.items
-        service_account_names_in_namespace: List[str] = list(map(lambda svcaccount: svcaccount.metadata.name, service_accounts_in_namespace))
+        service_account_names_in_namespace: List[str] = [svcaccount.metadata.name for svcaccount in service_accounts_in_namespace]
 
         if service_account_name not in service_account_names_in_namespace:
             service_account_metadata = {
@@ -344,28 +345,34 @@ def _create_service_account_if_not_exists(self, namespace: str, service_account_
             self.log.info(f"Created service account {service_account_name} in namespace {namespace}")
 
     def _forward_role_to_remote(self) -> None:
-        remote_cluster_roles: client.V1ClusterRoleList = client\
-            .RbacAuthorizationV1Api(api_client=kubernetes_client)\
-            .list_cluster_role()
-        remote_cluster_role_names = list(map(lambda role: role.metadata.name, remote_cluster_roles.items))
+        """If cluster role doesn't exist in target (remote) cluster, forward the one from the local cluster"""
 
+        # Get ClusterRoles in remote cluster
+        remote_cluster_roles: client.V1ClusterRoleList = (client
+                                                          .RbacAuthorizationV1Api(api_client=kubernetes_client)
+                                                          .list_cluster_role())
+        remote_cluster_role_names = [role.metadata.name for role in remote_cluster_roles.items]
+
+        # If the kernel ClusterRole does not exist in the remote cluster, grab it from the local cluster and
+        # create it in the remote. Allows us to configure the role via Helm.
         if kernel_cluster_role not in remote_cluster_role_names:
-            incluster_client = KUBERNETES_CLIENT_FACTORY.get_kubernetes_client(get_remote_if_available=False)
-            incluster_role: client.V1Role = client \
-                .RbacAuthorizationV1Api(api_client=incluster_client) \
-                .read_cluster_role(kernel_cluster_role)
+            incluster_client = KUBERNETES_CLIENT_FACTORY.get_kubernetes_client(get_remote_client=False)
+            incluster_role: client.V1Role = (client
+                .RbacAuthorizationV1Api(api_client=incluster_client)
+                .read_cluster_role(kernel_cluster_role))
 
+            # Mirror the local role but get rid of the resource version
             remote_cluster_role: client.V1Role = client.V1Role(
                 api_version=incluster_role.api_version,
                 kind=incluster_role.kind,
                 rules=incluster_role.rules,
                 metadata=incluster_role.metadata
             )
-
             remote_cluster_role.metadata.resource_version = None
 
             client.RbacAuthorizationV1Api(api_client=kubernetes_client).create_cluster_role(body=remote_cluster_role)
 
+            self.log.debug(f"Created kernel CluserRole with name {kernel_cluster_role}")
 
     def _create_role_binding(self, namespace: str, service_account_name: str) -> None:
         # Creates RoleBinding instance for the given namespace.  The role used will be the ClusterRole named by
@@ -395,7 +402,7 @@ def _create_role_binding(self, namespace: str, service_account_name: str) -> Non
 
         # If remote cluster is used, we need to create a cluster role on that cluster
         # Forward the role defined via helm to the remote cluster
-        if os.environ.get('EG_USE_REMOTE_CLUSTER'):
+        if os.getenv('EG_USE_REMOTE_CLUSTER'):
             self._forward_role_to_remote()
 
         client.RbacAuthorizationV1Api(api_client=kubernetes_client).create_namespaced_role_binding(

From 20a5c5664ef702096cf0373beaf13fe345544b55 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Wed, 25 Jan 2023 16:13:15 -0700
Subject: [PATCH 07/27] black fixes

---
 .../services/external/k8s_client.py           |  1 -
 .../services/external/k8s_client_factory.py   | 15 +++--
 .../services/processproxies/crd.py            |  8 ++-
 .../services/processproxies/k8s.py            | 66 +++++++++++--------
 4 files changed, 55 insertions(+), 35 deletions(-)

diff --git a/enterprise_gateway/services/external/k8s_client.py b/enterprise_gateway/services/external/k8s_client.py
index 2c6e157a9..d4815e605 100644
--- a/enterprise_gateway/services/external/k8s_client.py
+++ b/enterprise_gateway/services/external/k8s_client.py
@@ -4,4 +4,3 @@
 
 KUBERNETES_CLIENT_FACTORY = KubernetesClientFactory()
 kubernetes_client = KUBERNETES_CLIENT_FACTORY.get_kubernetes_client()
-
diff --git a/enterprise_gateway/services/external/k8s_client_factory.py b/enterprise_gateway/services/external/k8s_client_factory.py
index b2d8c8469..220fc977e 100644
--- a/enterprise_gateway/services/external/k8s_client_factory.py
+++ b/enterprise_gateway/services/external/k8s_client_factory.py
@@ -7,6 +7,7 @@
 
 class KubernetesClientFactory(SingletonConfigurable):
     """Manages kubernetes client creation from environment variables"""
+
     def __init__(self) -> None:
         """Maintain a single configuration object and populate based on environment"""
         super().__init__()
@@ -24,15 +25,19 @@ def get_kubernetes_client(self, get_remote_client: bool = True) -> client.ApiCli
         if os.getenv("KUBERNETES_SERVICE_HOST"):
             # Running inside cluster
             if os.getenv('EG_USE_REMOTE_CLUSTER') and get_remote_client:
-                kubeconfig_path = os.getenv('EG_REMOTE_CLUSTER_KUBECONFIG_PATH', '/etc/kube/config/')
+                kubeconfig_path = os.getenv(
+                    'EG_REMOTE_CLUSTER_KUBECONFIG_PATH', '/etc/kube/config/'
+                )
                 context = os.getenv('EG_REMOTE_CLUSTER_CONTEXT', None)
-                config.load_kube_config(client_configuration=kubernetes_config, config_file=kubeconfig_path, context=context)
+                config.load_kube_config(
+                    client_configuration=kubernetes_config,
+                    config_file=kubeconfig_path,
+                    context=context,
+                )
             else:
                 config.load_incluster_config(client_configuration=kubernetes_config)
         else:
             config.load_kube_config(client_configuration=kubernetes_config)
 
-        self.log.debug(
-            f"Created kubernetes client for host {kubernetes_config.host}"
-        )
+        self.log.debug(f"Created kubernetes client for host {kubernetes_config.host}")
         return client.ApiClient(kubernetes_config)
diff --git a/enterprise_gateway/services/processproxies/crd.py b/enterprise_gateway/services/processproxies/crd.py
index ead851f42..b9a3105b8 100644
--- a/enterprise_gateway/services/processproxies/crd.py
+++ b/enterprise_gateway/services/processproxies/crd.py
@@ -42,7 +42,9 @@ async def launch_process(
         use_remote_cluster = os.getenv("EG_USE_REMOTE_CLUSTER")
         if use_remote_cluster:
             kwargs["env"]["EG_USE_REMOTE_CLUSTER"] = 'true'
-            kwargs["env"]["EG_REMOTE_CLUSTER_KUBECONFIG_PATH"] = os.getenv("EG_REMOTE_CLUSTER_KUBECONFIG_PATH")
+            kwargs["env"]["EG_REMOTE_CLUSTER_KUBECONFIG_PATH"] = os.getenv(
+                "EG_REMOTE_CLUSTER_KUBECONFIG_PATH"
+            )
 
         await super().launch_process(kernel_cmd, **kwargs)
         return self
@@ -55,7 +57,9 @@ def delete_managed_object(self, termination_stati: list[str]) -> bool:
 
         Note: the caller is responsible for handling exceptions.
         """
-        delete_status = client.CustomObjectsApi(api_client=kubernetes_client).delete_namespaced_custom_object(
+        delete_status = client.CustomObjectsApi(
+            api_client=kubernetes_client
+        ).delete_namespaced_custom_object(
             self.group,
             self.version,
             self.kernel_namespace,
diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index 31914e6ed..849b79119 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -286,8 +286,9 @@ def _create_kernel_namespace(self, service_account_name: str) -> str:
 
             if os.getenv('EG_USE_REMOTE_CLUSTER'):
                 # If remote cluster is being used, service account may not be present, create before role binding
-                self._create_service_account_if_not_exists(namespace=namespace,
-                                                           service_account_name=service_account_name)
+                self._create_service_account_if_not_exists(
+                    namespace=namespace, service_account_name=service_account_name
+                )
 
             # Now create a RoleBinding for this namespace for the default ServiceAccount.  We'll reference
             # the ClusterRole, but that will only be applied for this namespace.  This prevents the need for
@@ -312,7 +313,9 @@ def _create_kernel_namespace(self, service_account_name: str) -> str:
                     body = client.V1DeleteOptions(
                         grace_period_seconds=0, propagation_policy="Background"
                     )
-                    client.CoreV1Api(api_client=kubernetes_client).delete_namespace(name=namespace, body=body)
+                    client.CoreV1Api(api_client=kubernetes_client).delete_namespace(
+                        name=namespace, body=body
+                    )
                     self.log.warning(f"Deleted kernel namespace: {namespace}")
                 else:
                     reason = f"Error occurred creating namespace '{namespace}': {err}"
@@ -320,57 +323,66 @@ def _create_kernel_namespace(self, service_account_name: str) -> str:
 
         return namespace
 
-    def _create_service_account_if_not_exists(self, namespace: str, service_account_name: str) -> None:
+    def _create_service_account_if_not_exists(
+        self, namespace: str, service_account_name: str
+    ) -> None:
         """If service account doesn't exist in target cluster, create one. Occurs if a remote cluster is being used."""
-        service_account_list_in_namespace: client.V1ServiceAccountList = (client
-                                                                          .CoreV1Api(api_client=kubernetes_client)
-                                                                          .list_namespaced_service_account(namespace=namespace))
+        service_account_list_in_namespace: client.V1ServiceAccountList = client.CoreV1Api(
+            api_client=kubernetes_client
+        ).list_namespaced_service_account(namespace=namespace)
 
-        service_accounts_in_namespace: List[client.V1ServiceAccount] = service_account_list_in_namespace.items
-        service_account_names_in_namespace: List[str] = [svcaccount.metadata.name for svcaccount in service_accounts_in_namespace]
+        service_accounts_in_namespace: List[
+            client.V1ServiceAccount
+        ] = service_account_list_in_namespace.items
+        service_account_names_in_namespace: List[str] = [
+            svcaccount.metadata.name for svcaccount in service_accounts_in_namespace
+        ]
 
         if service_account_name not in service_account_names_in_namespace:
-            service_account_metadata = {
-                "name": service_account_name
-            }
+            service_account_metadata = {"name": service_account_name}
             service_account_to_create: client.V1ServiceAccount = client.V1ServiceAccount(
-                kind="ServiceAccount",
-                metadata=service_account_metadata
+                kind="ServiceAccount", metadata=service_account_metadata
             )
 
-            client\
-                .CoreV1Api(api_client=kubernetes_client)\
-                .create_namespaced_service_account(namespace=namespace, body=service_account_to_create)
+            client.CoreV1Api(api_client=kubernetes_client).create_namespaced_service_account(
+                namespace=namespace, body=service_account_to_create
+            )
 
-            self.log.info(f"Created service account {service_account_name} in namespace {namespace}")
+            self.log.info(
+                f"Created service account {service_account_name} in namespace {namespace}"
+            )
 
     def _forward_role_to_remote(self) -> None:
         """If cluster role doesn't exist in target (remote) cluster, forward the one from the local cluster"""
 
         # Get ClusterRoles in remote cluster
-        remote_cluster_roles: client.V1ClusterRoleList = (client
-                                                          .RbacAuthorizationV1Api(api_client=kubernetes_client)
-                                                          .list_cluster_role())
+        remote_cluster_roles: client.V1ClusterRoleList = client.RbacAuthorizationV1Api(
+            api_client=kubernetes_client
+        ).list_cluster_role()
         remote_cluster_role_names = [role.metadata.name for role in remote_cluster_roles.items]
 
         # If the kernel ClusterRole does not exist in the remote cluster, grab it from the local cluster and
         # create it in the remote. Allows us to configure the role via Helm.
         if kernel_cluster_role not in remote_cluster_role_names:
-            incluster_client = KUBERNETES_CLIENT_FACTORY.get_kubernetes_client(get_remote_client=False)
-            incluster_role: client.V1Role = (client
-                .RbacAuthorizationV1Api(api_client=incluster_client)
-                .read_cluster_role(kernel_cluster_role))
+            incluster_client = KUBERNETES_CLIENT_FACTORY.get_kubernetes_client(
+                get_remote_client=False
+            )
+            incluster_role: client.V1Role = client.RbacAuthorizationV1Api(
+                api_client=incluster_client
+            ).read_cluster_role(kernel_cluster_role)
 
             # Mirror the local role but get rid of the resource version
             remote_cluster_role: client.V1Role = client.V1Role(
                 api_version=incluster_role.api_version,
                 kind=incluster_role.kind,
                 rules=incluster_role.rules,
-                metadata=incluster_role.metadata
+                metadata=incluster_role.metadata,
             )
             remote_cluster_role.metadata.resource_version = None
 
-            client.RbacAuthorizationV1Api(api_client=kubernetes_client).create_cluster_role(body=remote_cluster_role)
+            client.RbacAuthorizationV1Api(api_client=kubernetes_client).create_cluster_role(
+                body=remote_cluster_role
+            )
 
             self.log.debug(f"Created kernel CluserRole with name {kernel_cluster_role}")
 

From 0d43becc047deac7c1d20ef5287b609ded8b56b6 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Thu, 26 Jan 2023 16:04:15 -0700
Subject: [PATCH 08/27] Add remote autoconfiguration and role

---
 .../services/processproxies/k8s.py            | 61 ++++++++-----------
 .../templates/deployment.yaml                 | 58 ++++++++++++++++++
 .../templates/kernel-role.yaml                | 25 ++++++++
 .../templates/kubeconfig-configmap.yaml       |  3 +-
 .../templates/role-configmap.yaml             |  9 +++
 .../helm/enterprise-gateway/values.yaml       |  4 +-
 6 files changed, 123 insertions(+), 37 deletions(-)
 create mode 100644 etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
 create mode 100644 etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml

diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index 849b79119..043c45313 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -11,6 +11,7 @@
 
 import urllib3
 from kubernetes import client
+from kubernetes.utils.create_from_yaml import create_from_yaml
 
 from ..external.k8s_client import kubernetes_client, KUBERNETES_CLIENT_FACTORY
 from ..kernels.remotemanager import RemoteKernelManager
@@ -284,7 +285,7 @@ def _create_kernel_namespace(self, service_account_name: str) -> str:
             self.delete_kernel_namespace = True
             self.log.info(f"Created kernel namespace: {namespace}")
 
-            if os.getenv('EG_USE_REMOTE_CLUSTER'):
+            if os.getenv('EG_USE_REMOTE_CLUSTER') and os.getenv('EG_CREATE_REMOTE_SVC_ACCOUNT'):
                 # If remote cluster is being used, service account may not be present, create before role binding
                 self._create_service_account_if_not_exists(
                     namespace=namespace, service_account_name=service_account_name
@@ -352,8 +353,14 @@ def _create_service_account_if_not_exists(
                 f"Created service account {service_account_name} in namespace {namespace}"
             )
 
-    def _forward_role_to_remote(self) -> None:
-        """If cluster role doesn't exist in target (remote) cluster, forward the one from the local cluster"""
+    def _create_role_if_not_exists(self, namespace: str) -> None:
+        """If role doesn't exist in target cluster, create one. Occurs if a remote cluster is being used"""
+        role_yaml_path = os.getenv('EG_REMOTE_CLUSTER_ROLE_PATH')
+
+        role: client.V1Role
+        [role] = create_from_yaml(yaml_file=role_yaml_path, k8s_client=kubernetes_client)
+
+        role.metadata.namespace = namespace
 
         # Get ClusterRoles in remote cluster
         remote_cluster_roles: client.V1ClusterRoleList = client.RbacAuthorizationV1Api(
@@ -361,30 +368,10 @@ def _forward_role_to_remote(self) -> None:
         ).list_cluster_role()
         remote_cluster_role_names = [role.metadata.name for role in remote_cluster_roles.items]
 
-        # If the kernel ClusterRole does not exist in the remote cluster, grab it from the local cluster and
-        # create it in the remote. Allows us to configure the role via Helm.
+        # If the kernel ClusterRole does not exist in the remote cluster.
         if kernel_cluster_role not in remote_cluster_role_names:
-            incluster_client = KUBERNETES_CLIENT_FACTORY.get_kubernetes_client(
-                get_remote_client=False
-            )
-            incluster_role: client.V1Role = client.RbacAuthorizationV1Api(
-                api_client=incluster_client
-            ).read_cluster_role(kernel_cluster_role)
-
-            # Mirror the local role but get rid of the resource version
-            remote_cluster_role: client.V1Role = client.V1Role(
-                api_version=incluster_role.api_version,
-                kind=incluster_role.kind,
-                rules=incluster_role.rules,
-                metadata=incluster_role.metadata,
-            )
-            remote_cluster_role.metadata.resource_version = None
-
-            client.RbacAuthorizationV1Api(api_client=kubernetes_client).create_cluster_role(
-                body=remote_cluster_role
-            )
-
-            self.log.debug(f"Created kernel CluserRole with name {kernel_cluster_role}")
+            client.RbacAuthorizationV1Api(api_client=kubernetes_client).create_cluster_role(body=role)
+            self.log.debug(f"Created kernel role with name {kernel_cluster_role}")
 
     def _create_role_binding(self, namespace: str, service_account_name: str) -> None:
         # Creates RoleBinding instance for the given namespace.  The role used will be the ClusterRole named by
@@ -395,12 +382,23 @@ def _create_role_binding(self, namespace: str, service_account_name: str) -> Non
         # EG_DEFAULT_KERNEL_SERVICE_ACCOUNT_NAME, respectively.
         # We will not use a try/except clause here since _create_kernel_namespace will handle exceptions.
 
+        # If remote cluster is used, we need to create a role on that cluster
+        if os.getenv('EG_USE_REMOTE_CLUSTER') and os.getenv('EG_CREATE_ROLE_ON_REMOTE'):
+            self._create_role_if_not_exists(namespace=namespace)
+
         role_binding_name = kernel_cluster_role  # use same name for binding as cluster role
         labels = {"app": "enterprise-gateway", "component": "kernel", "kernel_id": self.kernel_id}
         binding_metadata = client.V1ObjectMeta(name=role_binding_name, labels=labels)
-        binding_role_ref = client.V1RoleRef(
-            api_group="", kind="ClusterRole", name=kernel_cluster_role
-        )
+
+        if os.getenv('EG_USE_REMOTE_CLUSTER'):
+            binding_role_ref = client.V1RoleRef(
+                api_group="", kind="Role", name=kernel_cluster_role
+            )
+        else:
+            binding_role_ref = client.V1RoleRef(
+                api_group="", kind="ClusterRole", name=kernel_cluster_role
+            )
+
         binding_subjects = client.V1Subject(
             api_group="", kind="ServiceAccount", name=service_account_name, namespace=namespace
         )
@@ -412,11 +410,6 @@ def _create_role_binding(self, namespace: str, service_account_name: str) -> Non
             subjects=[binding_subjects],
         )
 
-        # If remote cluster is used, we need to create a cluster role on that cluster
-        # Forward the role defined via helm to the remote cluster
-        if os.getenv('EG_USE_REMOTE_CLUSTER'):
-            self._forward_role_to_remote()
-
         client.RbacAuthorizationV1Api(api_client=kubernetes_client).create_namespaced_role_binding(
             namespace=namespace, body=body
         )
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
index 9f95cfce6..16f1203e8 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
@@ -109,6 +109,14 @@ spec:
         - name: EG_REMOTE_CLUSTER_CONTEXT
           value: {{ .Values.remoteKubeconfig.context }}
         {{- end}}
+        {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+        - name: EG_CREATE_REMOTE_SVC_ACCOUNT
+          value: "True"
+        - name: EG_CREATE_REMOTE_ROLE
+          value: "True"
+        - name: EG_REMOTE_CLUSTER_ROLE_PATH
+          value: "/etc/config/role/role.yaml"
+        {{- end }}
         {{- end }}
         # Optional authorization token passed in all requests
         {{- if .Values.authToken }}
@@ -138,19 +146,59 @@ spec:
         volumeMounts:
         - name: nfs-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
+        {{- if .Values.remoteKubeconfig.enabled }}
+        - name: kubeconfig
+          readOnly: true
+          mountPath: {{ .Values.remoteKubeconfig.configPath }}
+        {{- end}}
+        {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+        - name: role
+          readOnly: true
+          mountPath: "/etc/config/role"
+        {{- end }}
       volumes:
       - name: nfs-kernelspecs
         nfs:
           server: {{ .Values.nfs.internalServerIPAddress }}
           path: "/usr/local/share/jupyter/kernels"
+      {{- if .Values.remoteKubeconfig.enabled }}
+      - name: kubeconfig
+        configMap: 
+          name: {{ .Release.Name }}-configmap
+      {{- end}}
+      {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+      - name: role
+        configMap:
+          name: {{ .Release.name }}-configmap-role
+      {{- end }}
   {{- else if .Values.kernelspecsPvc.enabled }}
         volumeMounts:
         - name: pvc-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
+        {{- if .Values.remoteKubeconfig.enabled }}
+        - name: kubeconfig
+          readOnly: true
+          mountPath: {{ .Values.remoteKubeconfig.configPath }}
+        {{- end}}
+        {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+        - name: role
+          readOnly: true
+          mountPath: "/etc/config/role"
+        {{- end }}
       volumes:
       - name: pvc-kernelspecs
         persistentVolumeClaim:
           claimName: {{ .Values.kernelspecsPvc.name }}
+      {{- if .Values.remoteKubeconfig.enabled }}
+      - name: kubeconfig
+        configMap: 
+          name: {{ .Release.Name }}-configmap
+      {{- end}}
+      {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+      - name: role
+        configMap:
+          name: {{ .Release.name}}-configmap-role
+      {{- end }}
   {{- else if .Values.kernelspecs.image }}
         volumeMounts:
         - name: image-kernelspecs
@@ -160,6 +208,11 @@ spec:
           readOnly: true
           mountPath: {{ .Values.remoteKubeconfig.configPath }}
         {{- end}}
+        {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+        - name: role
+          readOnly: true
+          mountPath: "/etc/config/role"
+        {{- end }}
       volumes:
       - name: image-kernelspecs
         emptyDir:
@@ -169,6 +222,11 @@ spec:
         configMap: 
           name: {{ .Release.Name }}-configmap
       {{- end}}
+      {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+      - name: role
+        configMap:
+          name: {{ .Release.Name }}-configmap-role
+      {{- end }}
 
   {{- if .Values.deployment.tolerations }}
       tolerations:
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
new file mode 100644
index 000000000..f8a583044
--- /dev/null
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
@@ -0,0 +1,25 @@
+{{- define "kernel-role" }}
+{{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  # Referenced by EG_KERNEL_CLUSTER_ROLE in the Deployment
+  name: kernel-controller
+  labels:
+    app: enterprise-gateway
+    component: kernel
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "watch", "list", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["list", "create"]
+  - apiGroups: [""]
+    resources: ["services", "persistentvolumeclaims"]
+    verbs: ["list"]
+{{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
index be9741102..9f38466a5 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
@@ -4,6 +4,5 @@ kind: ConfigMap
 metadata:
   name: {{ .Release.Name }}-configmap
 data:
-  config: |
-{{ .Files.Get "kubeconfig" | indent 6 }}
+  role: {{- (tpl (.Files.Glob "config/*").AsConfig . ) | nindent 2 }}
 {{- end }}
\ No newline at end of file
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
new file mode 100644
index 000000000..9bbfb0c71
--- /dev/null
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
@@ -0,0 +1,9 @@
+{{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-configmap-role
+data:
+  role.yaml: |
+    {{- include "kernel-role" . | indent 4 }}
+{{- end }}
\ No newline at end of file
diff --git a/etc/kubernetes/helm/enterprise-gateway/values.yaml b/etc/kubernetes/helm/enterprise-gateway/values.yaml
index 86879c9e3..7ed996fba 100644
--- a/etc/kubernetes/helm/enterprise-gateway/values.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/values.yaml
@@ -184,5 +184,7 @@ remoteKubeconfig:
   # Set to enable to point k8s client to remote cluster
   enabled: false
   # Mount config at this path inside pod
-  configPath:
+  configPath: /etc/kube/config/
+  # Automatically create service account and role in remote cluster
+  autoconfigureRemote: false
 

From 875fafaab3e8e0403b8d79e13a445896fcbdea94 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Thu, 26 Jan 2023 16:14:49 -0700
Subject: [PATCH 09/27] Fix env var checking

---
 .../services/external/k8s_client_factory.py              | 3 ++-
 enterprise_gateway/services/processproxies/k8s.py        | 9 +++++----
 enterprise_gateway/services/utils/__init__.py            | 0
 enterprise_gateway/services/utils/envutils.py            | 5 +++++
 4 files changed, 12 insertions(+), 5 deletions(-)
 create mode 100644 enterprise_gateway/services/utils/__init__.py
 create mode 100644 enterprise_gateway/services/utils/envutils.py

diff --git a/enterprise_gateway/services/external/k8s_client_factory.py b/enterprise_gateway/services/external/k8s_client_factory.py
index 220fc977e..aee532c32 100644
--- a/enterprise_gateway/services/external/k8s_client_factory.py
+++ b/enterprise_gateway/services/external/k8s_client_factory.py
@@ -3,6 +3,7 @@
 
 from kubernetes import client, config
 from traitlets.config import SingletonConfigurable
+from ..utils.envutils import is_env_true
 
 
 class KubernetesClientFactory(SingletonConfigurable):
@@ -24,7 +25,7 @@ def get_kubernetes_client(self, get_remote_client: bool = True) -> client.ApiCli
         kubernetes_config: client.Configuration = client.Configuration()
         if os.getenv("KUBERNETES_SERVICE_HOST"):
             # Running inside cluster
-            if os.getenv('EG_USE_REMOTE_CLUSTER') and get_remote_client:
+            if is_env_true('EG_USE_REMOTE_CLUSTER') and get_remote_client:
                 kubeconfig_path = os.getenv(
                     'EG_REMOTE_CLUSTER_KUBECONFIG_PATH', '/etc/kube/config/'
                 )
diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index 043c45313..e44dcb5fa 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -13,7 +13,8 @@
 from kubernetes import client
 from kubernetes.utils.create_from_yaml import create_from_yaml
 
-from ..external.k8s_client import kubernetes_client, KUBERNETES_CLIENT_FACTORY
+from ..external.k8s_client import kubernetes_client
+from ..utils.envutils import is_env_true
 from ..kernels.remotemanager import RemoteKernelManager
 from ..sessions.kernelsessionmanager import KernelSessionManager
 from .container import ContainerProcessProxy
@@ -285,7 +286,7 @@ def _create_kernel_namespace(self, service_account_name: str) -> str:
             self.delete_kernel_namespace = True
             self.log.info(f"Created kernel namespace: {namespace}")
 
-            if os.getenv('EG_USE_REMOTE_CLUSTER') and os.getenv('EG_CREATE_REMOTE_SVC_ACCOUNT'):
+            if is_env_true('EG_USE_REMOTE_CLUSTER') and os.getenv('EG_CREATE_REMOTE_SVC_ACCOUNT'):
                 # If remote cluster is being used, service account may not be present, create before role binding
                 self._create_service_account_if_not_exists(
                     namespace=namespace, service_account_name=service_account_name
@@ -383,14 +384,14 @@ def _create_role_binding(self, namespace: str, service_account_name: str) -> Non
         # We will not use a try/except clause here since _create_kernel_namespace will handle exceptions.
 
         # If remote cluster is used, we need to create a role on that cluster
-        if os.getenv('EG_USE_REMOTE_CLUSTER') and os.getenv('EG_CREATE_ROLE_ON_REMOTE'):
+        if is_env_true('EG_USE_REMOTE_CLUSTER') and os.getenv('EG_CREATE_ROLE_ON_REMOTE'):
             self._create_role_if_not_exists(namespace=namespace)
 
         role_binding_name = kernel_cluster_role  # use same name for binding as cluster role
         labels = {"app": "enterprise-gateway", "component": "kernel", "kernel_id": self.kernel_id}
         binding_metadata = client.V1ObjectMeta(name=role_binding_name, labels=labels)
 
-        if os.getenv('EG_USE_REMOTE_CLUSTER'):
+        if is_env_true('EG_USE_REMOTE_CLUSTER'):
             binding_role_ref = client.V1RoleRef(
                 api_group="", kind="Role", name=kernel_cluster_role
             )
diff --git a/enterprise_gateway/services/utils/__init__.py b/enterprise_gateway/services/utils/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/enterprise_gateway/services/utils/envutils.py b/enterprise_gateway/services/utils/envutils.py
new file mode 100644
index 000000000..01e80b4b5
--- /dev/null
+++ b/enterprise_gateway/services/utils/envutils.py
@@ -0,0 +1,5 @@
+import os
+
+
+def is_env_true(env_variable_name: str):
+    return bool(os.getenv(env_variable_name, "False").lower() == "true")
\ No newline at end of file

From 5d1d732a094d5c9ed38138e460591ad51e0ba567 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Fri, 27 Jan 2023 14:25:12 -0700
Subject: [PATCH 10/27] move to namespaced roles for remote clusters

---
 .../services/external/k8s_client_factory.py   |  2 +-
 .../services/processproxies/k8s.py            | 28 ++++++++-----------
 .../templates/deployment.yaml                 |  6 ++--
 .../templates/kubeconfig-configmap.yaml       |  2 +-
 4 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/enterprise_gateway/services/external/k8s_client_factory.py b/enterprise_gateway/services/external/k8s_client_factory.py
index aee532c32..e2da932ba 100644
--- a/enterprise_gateway/services/external/k8s_client_factory.py
+++ b/enterprise_gateway/services/external/k8s_client_factory.py
@@ -27,7 +27,7 @@ def get_kubernetes_client(self, get_remote_client: bool = True) -> client.ApiCli
             # Running inside cluster
             if is_env_true('EG_USE_REMOTE_CLUSTER') and get_remote_client:
                 kubeconfig_path = os.getenv(
-                    'EG_REMOTE_CLUSTER_KUBECONFIG_PATH', '/etc/kube/config/'
+                    'EG_REMOTE_CLUSTER_KUBECONFIG_PATH', '/etc/kube/config/kubeconfig'
                 )
                 context = os.getenv('EG_REMOTE_CLUSTER_CONTEXT', None)
                 config.load_kube_config(
diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index e44dcb5fa..c0eae77e5 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -10,8 +10,9 @@
 from typing import Any, List
 
 import urllib3
+import yaml
 from kubernetes import client
-from kubernetes.utils.create_from_yaml import create_from_yaml
+from kubernetes.utils.create_from_yaml import create_from_yaml_single_item
 
 from ..external.k8s_client import kubernetes_client
 from ..utils.envutils import is_env_true
@@ -358,21 +359,18 @@ def _create_role_if_not_exists(self, namespace: str) -> None:
         """If role doesn't exist in target cluster, create one. Occurs if a remote cluster is being used"""
         role_yaml_path = os.getenv('EG_REMOTE_CLUSTER_ROLE_PATH')
 
-        role: client.V1Role
-        [role] = create_from_yaml(yaml_file=role_yaml_path, k8s_client=kubernetes_client)
-
-        role.metadata.namespace = namespace
-
-        # Get ClusterRoles in remote cluster
-        remote_cluster_roles: client.V1ClusterRoleList = client.RbacAuthorizationV1Api(
+        # Get Roles in remote cluster
+        remote_cluster_roles: client.V1RoleList = client.RbacAuthorizationV1Api(
             api_client=kubernetes_client
-        ).list_cluster_role()
+        ).list_namespaced_role(namespace=namespace)
         remote_cluster_role_names = [role.metadata.name for role in remote_cluster_roles.items]
 
-        # If the kernel ClusterRole does not exist in the remote cluster.
+        # If the kernel Role does not exist in the remote cluster.
         if kernel_cluster_role not in remote_cluster_role_names:
-            client.RbacAuthorizationV1Api(api_client=kubernetes_client).create_cluster_role(body=role)
-            self.log.debug(f"Created kernel role with name {kernel_cluster_role}")
+            with open(role_yaml_path, 'r') as f:
+                role_yaml = yaml.safe_load(f)
+                role_yaml["metadata"]["namespace"] = namespace
+                create_from_yaml_single_item(yml_object=role_yaml, k8s_client=kubernetes_client)
 
     def _create_role_binding(self, namespace: str, service_account_name: str) -> None:
         # Creates RoleBinding instance for the given namespace.  The role used will be the ClusterRole named by
@@ -383,15 +381,13 @@ def _create_role_binding(self, namespace: str, service_account_name: str) -> Non
         # EG_DEFAULT_KERNEL_SERVICE_ACCOUNT_NAME, respectively.
         # We will not use a try/except clause here since _create_kernel_namespace will handle exceptions.
 
-        # If remote cluster is used, we need to create a role on that cluster
-        if is_env_true('EG_USE_REMOTE_CLUSTER') and os.getenv('EG_CREATE_ROLE_ON_REMOTE'):
-            self._create_role_if_not_exists(namespace=namespace)
-
         role_binding_name = kernel_cluster_role  # use same name for binding as cluster role
         labels = {"app": "enterprise-gateway", "component": "kernel", "kernel_id": self.kernel_id}
         binding_metadata = client.V1ObjectMeta(name=role_binding_name, labels=labels)
 
+        # If remote cluster is used, we need to create a role on that cluster
         if is_env_true('EG_USE_REMOTE_CLUSTER'):
+            self._create_role_if_not_exists(namespace=namespace)
             binding_role_ref = client.V1RoleRef(
                 api_group="", kind="Role", name=kernel_cluster_role
             )
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
index 16f1203e8..006156695 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
@@ -61,7 +61,7 @@ spec:
           - name: EG_USE_REMOTE_CLUSTER
             value: "True"
           - name: EG_REMOTE_CLUSTER_KUBECONFIG_PATH
-            value: {{ .Values.remoteKubeconfig.configPath }}config
+            value: {{ .Values.remoteKubeconfig.configPath }}{{ .Values.remoteKubeconfig.configFilename }}
         {{- end }}
   {{- end }}
       containers:
@@ -102,7 +102,7 @@ spec:
         - name: EG_USE_REMOTE_CLUSTER
           value: "True"
         - name: EG_REMOTE_CLUSTER_KUBECONFIG_PATH
-          value: {{ .Values.remoteKubeconfig.configPath }}config
+          value: {{ .Values.remoteKubeconfig.configPath }}{{ .Values.remoteKubeconfig.configFilename }}
         - name: EG_DEFAULT_KERNEL_SERVICE_ACCOUNT_NAME
           value: kernel-svcaccount
         {{- if .Values.remoteKubeconfig.context }}
@@ -112,8 +112,6 @@ spec:
         {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
         - name: EG_CREATE_REMOTE_SVC_ACCOUNT
           value: "True"
-        - name: EG_CREATE_REMOTE_ROLE
-          value: "True"
         - name: EG_REMOTE_CLUSTER_ROLE_PATH
           value: "/etc/config/role/role.yaml"
         {{- end }}
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
index 9f38466a5..56ab05c8a 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
@@ -4,5 +4,5 @@ kind: ConfigMap
 metadata:
   name: {{ .Release.Name }}-configmap
 data:
-  role: {{- (tpl (.Files.Glob "config/*").AsConfig . ) | nindent 2 }}
+  {{- (tpl (.Files.Glob "config/*").AsConfig . ) | nindent 2 }}
 {{- end }}
\ No newline at end of file

From 19f92c75402b8d0b1818d580c348468feb069915 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Fri, 27 Jan 2023 15:45:04 -0700
Subject: [PATCH 11/27] Formatting and comments

---
 .../services/external/k8s_client.py           |  1 -
 .../services/external/k8s_client_factory.py   |  1 +
 .../services/processproxies/crd.py            |  2 +-
 .../services/processproxies/k8s.py            | 23 +++++++++++--------
 enterprise_gateway/services/utils/envutils.py |  2 +-
 .../kubernetes/scripts/launch_kubernetes.py   | 13 +++++++----
 .../scripts/launch_custom_resource.py         |  1 +
 7 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/enterprise_gateway/services/external/k8s_client.py b/enterprise_gateway/services/external/k8s_client.py
index d4815e605..7d93188ef 100644
--- a/enterprise_gateway/services/external/k8s_client.py
+++ b/enterprise_gateway/services/external/k8s_client.py
@@ -1,5 +1,4 @@
 """Instantiates a static global factory and a single atomic client"""
-import os
 from enterprise_gateway.services.external.k8s_client_factory import KubernetesClientFactory
 
 KUBERNETES_CLIENT_FACTORY = KubernetesClientFactory()
diff --git a/enterprise_gateway/services/external/k8s_client_factory.py b/enterprise_gateway/services/external/k8s_client_factory.py
index e2da932ba..46b79130d 100644
--- a/enterprise_gateway/services/external/k8s_client_factory.py
+++ b/enterprise_gateway/services/external/k8s_client_factory.py
@@ -3,6 +3,7 @@
 
 from kubernetes import client, config
 from traitlets.config import SingletonConfigurable
+
 from ..utils.envutils import is_env_true
 
 
diff --git a/enterprise_gateway/services/processproxies/crd.py b/enterprise_gateway/services/processproxies/crd.py
index b9a3105b8..a8ff3426a 100644
--- a/enterprise_gateway/services/processproxies/crd.py
+++ b/enterprise_gateway/services/processproxies/crd.py
@@ -4,8 +4,8 @@
 
 from __future__ import annotations
 
-from typing import Any
 import os
+from typing import Any
 
 from kubernetes import client
 
diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index c0eae77e5..6ea3e6fdf 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -7,7 +7,7 @@
 import logging
 import os
 import re
-from typing import Any, List
+from typing import Any
 
 import urllib3
 import yaml
@@ -15,9 +15,9 @@
 from kubernetes.utils.create_from_yaml import create_from_yaml_single_item
 
 from ..external.k8s_client import kubernetes_client
-from ..utils.envutils import is_env_true
 from ..kernels.remotemanager import RemoteKernelManager
 from ..sessions.kernelsessionmanager import KernelSessionManager
+from ..utils.envutils import is_env_true
 from .container import ContainerProcessProxy
 
 urllib3.disable_warnings()
@@ -230,7 +230,6 @@ def _determine_kernel_pod_name(self, **kwargs: dict[str, Any] | None) -> str:
         return pod_name
 
     def _determine_kernel_namespace(self, **kwargs: dict[str, Any] | None) -> str:
-
         # Since we need the service account name regardless of whether we're creating the namespace or not,
         # get it now.
         service_account_name = KubernetesProcessProxy._determine_kernel_service_account_name(
@@ -287,8 +286,9 @@ def _create_kernel_namespace(self, service_account_name: str) -> str:
             self.delete_kernel_namespace = True
             self.log.info(f"Created kernel namespace: {namespace}")
 
+            # If remote cluster is being used, service account may not be present, create before role binding
+            # If creating service account is disabled, operator must manually create svc account
             if is_env_true('EG_USE_REMOTE_CLUSTER') and os.getenv('EG_CREATE_REMOTE_SVC_ACCOUNT'):
-                # If remote cluster is being used, service account may not be present, create before role binding
                 self._create_service_account_if_not_exists(
                     namespace=namespace, service_account_name=service_account_name
                 )
@@ -334,10 +334,10 @@ def _create_service_account_if_not_exists(
             api_client=kubernetes_client
         ).list_namespaced_service_account(namespace=namespace)
 
-        service_accounts_in_namespace: List[
+        service_accounts_in_namespace: list[
             client.V1ServiceAccount
         ] = service_account_list_in_namespace.items
-        service_account_names_in_namespace: List[str] = [
+        service_account_names_in_namespace: list[str] = [
             svcaccount.metadata.name for svcaccount in service_accounts_in_namespace
         ]
 
@@ -367,11 +367,15 @@ def _create_role_if_not_exists(self, namespace: str) -> None:
 
         # If the kernel Role does not exist in the remote cluster.
         if kernel_cluster_role not in remote_cluster_role_names:
-            with open(role_yaml_path, 'r') as f:
+            with open(role_yaml_path) as f:
                 role_yaml = yaml.safe_load(f)
                 role_yaml["metadata"]["namespace"] = namespace
                 create_from_yaml_single_item(yml_object=role_yaml, k8s_client=kubernetes_client)
 
+            self.log.info(f"Created role {kernel_cluster_role} in namespace {namespace}")
+        else:
+            self.log.info(f"Found role {kernel_cluster_role} in namespace {namespace}")
+
     def _create_role_binding(self, namespace: str, service_account_name: str) -> None:
         # Creates RoleBinding instance for the given namespace.  The role used will be the ClusterRole named by
         # EG_KERNEL_CLUSTER_ROLE.
@@ -388,9 +392,8 @@ def _create_role_binding(self, namespace: str, service_account_name: str) -> Non
         # If remote cluster is used, we need to create a role on that cluster
         if is_env_true('EG_USE_REMOTE_CLUSTER'):
             self._create_role_if_not_exists(namespace=namespace)
-            binding_role_ref = client.V1RoleRef(
-                api_group="", kind="Role", name=kernel_cluster_role
-            )
+            # We use namespaced roles on remote clusters rather than a ClusterRole
+            binding_role_ref = client.V1RoleRef(api_group="", kind="Role", name=kernel_cluster_role)
         else:
             binding_role_ref = client.V1RoleRef(
                 api_group="", kind="ClusterRole", name=kernel_cluster_role
diff --git a/enterprise_gateway/services/utils/envutils.py b/enterprise_gateway/services/utils/envutils.py
index 01e80b4b5..b8d97be2c 100644
--- a/enterprise_gateway/services/utils/envutils.py
+++ b/enterprise_gateway/services/utils/envutils.py
@@ -2,4 +2,4 @@
 
 
 def is_env_true(env_variable_name: str):
-    return bool(os.getenv(env_variable_name, "False").lower() == "true")
\ No newline at end of file
+    return bool(os.getenv(env_variable_name, "False").lower() == "true")
diff --git a/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py b/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py
index 16ae0288e..f6a803ad8 100644
--- a/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py
+++ b/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py
@@ -9,9 +9,10 @@
 import yaml
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 from kubernetes import client
-from enterprise_gateway.services.external.k8s_client import kubernetes_client
 from kubernetes.client.rest import ApiException
 
+from enterprise_gateway.services.external.k8s_client import kubernetes_client
+
 urllib3.disable_warnings()
 
 KERNEL_POD_TEMPLATE_PATH = "/kernel-pod.yaml.j2"
@@ -149,9 +150,9 @@ def launch_kubernetes_kernel(
                 pod_template = extend_pod_env(k8s_obj)
                 if pod_template_file is None:
                     try:
-                        pod_created = client.CoreV1Api(api_client=kubernetes_client).create_namespaced_pod(
-                            body=k8s_obj, namespace=kernel_namespace
-                        )
+                        pod_created = client.CoreV1Api(
+                            api_client=kubernetes_client
+                        ).create_namespaced_pod(body=k8s_obj, namespace=kernel_namespace)
                     except ApiException as exc:
                         if _parse_k8s_exception(exc) == K8S_ALREADY_EXIST_REASON:
                             pod_created = (
@@ -185,7 +186,9 @@ def launch_kubernetes_kernel(
                             raise exc
             elif k8s_obj["kind"] == "PersistentVolume":
                 if pod_template_file is None:
-                    client.CoreV1Api(api_client=kubernetes_client).create_persistent_volume(body=k8s_obj)
+                    client.CoreV1Api(api_client=kubernetes_client).create_persistent_volume(
+                        body=k8s_obj
+                    )
             elif k8s_obj["kind"] == "Service":
                 if pod_template_file is None:
                     if pod_created is not None:
diff --git a/etc/kernel-launchers/operators/scripts/launch_custom_resource.py b/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
index d634ecf60..e1093b315 100644
--- a/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
+++ b/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
@@ -8,6 +8,7 @@
 import yaml
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 from kubernetes import client
+
 from enterprise_gateway.services.external.k8s_client import kubernetes_client
 
 urllib3.disable_warnings()

From e0df4e2c33ceb7f77bdb487a0f6aefcf7aaedd9d Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Fri, 27 Jan 2023 16:08:27 -0700
Subject: [PATCH 12/27] Add documentation and cleanup

---
 docs/source/operators/deploy-kubernetes.md    | 24 +++++++++
 .../templates/deployment.yaml                 | 50 +++++++++----------
 .../templates/eg-clusterrole.yaml             |  4 +-
 .../templates/kernel-role.yaml                |  2 +-
 .../templates/kubeconfig-configmap.yaml       |  2 +-
 .../templates/role-configmap.yaml             |  4 +-
 .../helm/enterprise-gateway/values.yaml       |  5 +-
 7 files changed, 59 insertions(+), 32 deletions(-)

diff --git a/docs/source/operators/deploy-kubernetes.md b/docs/source/operators/deploy-kubernetes.md
index 5070222a0..9fa02e947 100644
--- a/docs/source/operators/deploy-kubernetes.md
+++ b/docs/source/operators/deploy-kubernetes.md
@@ -520,6 +520,10 @@ can override them with helm's `--set` or `--values` options. Always use `--set`
 | `kip.pullPolicy`                           | Determines whether the Kernel Image Puller will pull kernel images it has previously pulled (`Always`) or only those it hasn't yet pulled (`IfNotPresent`)                                                                                       | `IfNotPresent`                                                                 |
 | `kip.criSocket`                            | The container runtime interface socket, use `/run/containerd/containerd.sock` for containerd installations                                                                                                                                       | `/var/run/docker.sock`                                                         |
 | `kip.defaultContainerRegistry`             | Prefix to use if a registry is not already specified on image name (e.g., elyra/kernel-py:VERSION)                                                                                                                                               | `docker.io`                                                                    |
+ | `multicluster.enable` | Launch kernels in a remote cluster. Used for multi-cluster environments. **Must place a kubeconfig file in the `config/` folder of the helm chart**. | `false`
+ | `multicluster.configPath` | Path to mount kubeconfig at | `/etc/kube/config`|
+ |`multicluster.configFilename` | Filename to kubeconfig file inside `config/` directory of chart | `kubeconfig`|
+ | `multicluster.autoconfigureRemote` | Automatically create service account in remote cluster | `false`
 
 ## Uninstalling Enterprise Gateway
 
@@ -955,6 +959,26 @@ Of particular importance is the mapping to port `8888` (e.g.,`32422`). If you ar
 
 The value of the `JUPYTER_GATEWAY_URL` used by the gateway-enabled Notebook server will vary depending on whether you choose to define an external IP or not. If and external IP is defined, you'll set `JUPYTER_GATEWAY_URL=<externalIP>:8888` else you'll set `JUPYTER_GATEWAY_URL=<k8s-master>:32422` **but also need to restart clients each time Enterprise Gateway is started.** As a result, use of the `externalIPs:` value is highly recommended.
 
+## Multi-Cluster Environments
+
+### Overview
+With `multicluster.enabled` set to `true`, Enterprise Gateway can be used on multi-cluster environments where the jupyter enterprise gateway pods and kernel pods are launched on separate clusters. To configure this:
+
+1. Ensure your two clusters have interconnceted networks. Pods in the two clusters must be able to communicate with each other over pod IP alone.
+2. Provide a kubeconfig file for use in the `config/` subdirectory of `etc/kubernetes/helm/enterprise-gateway` chart. 
+3. Set `multicluster.enabled` to `true`. 
+
+Enterprise Gateway will now launch kernel pods in whichever cluster you have set to default in your kubeconfig. 
+
+### Resources in Remote Clusters
+ For Enterprise Gateway to work accross clusters, Enterprise Gateway must create the following resources in the cluster your kernels will be launched on. 
+ - The kernel resource.
+ - A service account for the kernel pods (if `multicluster.autoconfigureRemote` is set to `true`).
+ - A namespaced role for the namespace where your kernel pods will be launched.
+ - A role binding between your namespaced role and your service account. 
+
+ The role resource is defined in the `templates/kernel-role.yaml` template of the helm chart. Permissions can be set there. 
+ 
 ## Kubernetes Tips
 
 The following items illustrate some useful commands for navigating Enterprise Gateway within a kubernetes environment.
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
index 006156695..9d7b5cff8 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
@@ -44,24 +44,24 @@ spec:
         volumeMounts:
         - name: image-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
-        {{- if .Values.remoteKubeconfig.enabled }}
+        {{- if .Values.multicluster.enabled }}
         - name: kubeconfig
           readOnly: true
-          mountPath: {{ .Values.remoteKubeconfig.configPath }}
+          mountPath: {{ .Values.multicluster.configPath }}
         {{- end}}
         volumes:
-        {{- if .Values.remoteKubeconfig.enabled }}
+        {{- if .Values.multicluster.enabled }}
         - name: kubeconfig
           configMap: 
             name: {{ .Release.Name }}-configmap
         {{- end}}
         env:
         # Set environment variables to point k8s client to kubeconfig
-        {{- if .Values.remoteKubeconfig.enabled }}
+        {{- if .Values.multicluster.enabled }}
           - name: EG_USE_REMOTE_CLUSTER
             value: "True"
           - name: EG_REMOTE_CLUSTER_KUBECONFIG_PATH
-            value: {{ .Values.remoteKubeconfig.configPath }}{{ .Values.remoteKubeconfig.configFilename }}
+            value: {{ .Values.multicluster.configPath }}{{ .Values.multicluster.configFilename }}
         {{- end }}
   {{- end }}
       containers:
@@ -98,18 +98,18 @@ spec:
         - name: EG_DEFAULT_KERNEL_NAME
           value: {{ .Values.kernel.defaultKernelName }}
         # Set environment variables to point k8s client to kubeconfig
-        {{- if .Values.remoteKubeconfig.enabled }}
+        {{- if .Values.multicluster.enabled }}
         - name: EG_USE_REMOTE_CLUSTER
           value: "True"
         - name: EG_REMOTE_CLUSTER_KUBECONFIG_PATH
-          value: {{ .Values.remoteKubeconfig.configPath }}{{ .Values.remoteKubeconfig.configFilename }}
+          value: {{ .Values.multicluster.configPath }}{{ .Values.multicluster.configFilename }}
         - name: EG_DEFAULT_KERNEL_SERVICE_ACCOUNT_NAME
           value: kernel-svcaccount
-        {{- if .Values.remoteKubeconfig.context }}
+        {{- if .Values.multicluster.context }}
         - name: EG_REMOTE_CLUSTER_CONTEXT
-          value: {{ .Values.remoteKubeconfig.context }}
+          value: {{ .Values.multicluster.context }}
         {{- end}}
-        {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+        {{- if .Values.multicluster.autoconfigureRemote }}
         - name: EG_CREATE_REMOTE_SVC_ACCOUNT
           value: "True"
         - name: EG_REMOTE_CLUSTER_ROLE_PATH
@@ -144,12 +144,12 @@ spec:
         volumeMounts:
         - name: nfs-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
-        {{- if .Values.remoteKubeconfig.enabled }}
+        {{- if .Values.multicluster.enabled }}
         - name: kubeconfig
           readOnly: true
-          mountPath: {{ .Values.remoteKubeconfig.configPath }}
+          mountPath: {{ .Values.multicluster.configPath }}
         {{- end}}
-        {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+        {{- if .Values.multicluster.autoconfigureRemote }}
         - name: role
           readOnly: true
           mountPath: "/etc/config/role"
@@ -159,12 +159,12 @@ spec:
         nfs:
           server: {{ .Values.nfs.internalServerIPAddress }}
           path: "/usr/local/share/jupyter/kernels"
-      {{- if .Values.remoteKubeconfig.enabled }}
+      {{- if .Values.multicluster.enabled }}
       - name: kubeconfig
         configMap: 
           name: {{ .Release.Name }}-configmap
       {{- end}}
-      {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+      {{- if .Values.multicluster.autoconfigureRemote }}
       - name: role
         configMap:
           name: {{ .Release.name }}-configmap-role
@@ -173,12 +173,12 @@ spec:
         volumeMounts:
         - name: pvc-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
-        {{- if .Values.remoteKubeconfig.enabled }}
+        {{- if .Values.multicluster.enabled }}
         - name: kubeconfig
           readOnly: true
-          mountPath: {{ .Values.remoteKubeconfig.configPath }}
+          mountPath: {{ .Values.multicluster.configPath }}
         {{- end}}
-        {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+        {{- if .Values.multicluster.autoconfigureRemote }}
         - name: role
           readOnly: true
           mountPath: "/etc/config/role"
@@ -187,12 +187,12 @@ spec:
       - name: pvc-kernelspecs
         persistentVolumeClaim:
           claimName: {{ .Values.kernelspecsPvc.name }}
-      {{- if .Values.remoteKubeconfig.enabled }}
+      {{- if .Values.multicluster.enabled }}
       - name: kubeconfig
         configMap: 
           name: {{ .Release.Name }}-configmap
       {{- end}}
-      {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+      {{- if .Values.multicluster.autoconfigureRemote }}
       - name: role
         configMap:
           name: {{ .Release.name}}-configmap-role
@@ -201,12 +201,12 @@ spec:
         volumeMounts:
         - name: image-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
-        {{- if .Values.remoteKubeconfig.enabled }}
+        {{- if .Values.multicluster.enabled }}
         - name: kubeconfig
           readOnly: true
-          mountPath: {{ .Values.remoteKubeconfig.configPath }}
+          mountPath: {{ .Values.multicluster.configPath }}
         {{- end}}
-        {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+        {{- if .Values.multicluster.autoconfigureRemote }}
         - name: role
           readOnly: true
           mountPath: "/etc/config/role"
@@ -215,12 +215,12 @@ spec:
       - name: image-kernelspecs
         emptyDir:
           medium: Memory
-      {{- if .Values.remoteKubeconfig.enabled }}
+      {{- if .Values.multicluster.enabled }}
       - name: kubeconfig
         configMap: 
           name: {{ .Release.Name }}-configmap
       {{- end}}
-      {{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+      {{- if .Values.multicluster.autoconfigureRemote }}
       - name: role
         configMap:
           name: {{ .Release.Name }}-configmap-role
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
index d38607dbb..55c703441 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
@@ -23,8 +23,8 @@ rules:
   - apiGroups: ["sparkoperator.k8s.io"]
     resources: ["sparkapplications", "sparkapplications/status", "scheduledsparkapplications", "scheduledsparkapplications/status"]
     verbs: ["get", "watch", "list", "create", "delete"]
-  # If remoteKubeconfig used, need to be able to read and forward service account
-  {{- if .Values.remoteKubeconfig.enabled }}
+  # If multicluster used, need to be able to read and forward service account
+  {{- if .Values.multicluster.enabled }}
   - apiGroups: ["rbac.authorization.k8s.io"]
     resources: ["clusterroles"]
     verbs: ["get", "create"]
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
index f8a583044..4dc20bb61 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
@@ -1,5 +1,5 @@
 {{- define "kernel-role" }}
-{{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+{{- if .Values.multicluster.autoconfigureRemote }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
index 56ab05c8a..8f8070207 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.remoteKubeconfig.enabled }}
+{{- if .Values.multicluster.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
index 9bbfb0c71..3b53b1f6d 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
@@ -1,9 +1,11 @@
-{{- if .Values.remoteKubeconfig.autoconfigureRemote }}
+{{- if .Values.multicluster.autoconfigureRemote }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ .Release.Name }}-configmap-role
 data:
+  # Pass in Role from helm chart into configmap that k8s client uses to create role in remote cluster
+  # Allows for Role permissions to be defined via helm chart, not source code
   role.yaml: |
     {{- include "kernel-role" . | indent 4 }}
 {{- end }}
\ No newline at end of file
diff --git a/etc/kubernetes/helm/enterprise-gateway/values.yaml b/etc/kubernetes/helm/enterprise-gateway/values.yaml
index 7ed996fba..811783bca 100644
--- a/etc/kubernetes/helm/enterprise-gateway/values.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/values.yaml
@@ -180,11 +180,12 @@ kip:
   #     cpu: 1
   #     memory: 1Gi
 
-remoteKubeconfig:
+multicluster:
   # Set to enable to point k8s client to remote cluster
   enabled: false
   # Mount config at this path inside pod
   configPath: /etc/kube/config/
+  # Filename of kubeconfig
+  configFilename: kubeconfig
   # Automatically create service account and role in remote cluster
   autoconfigureRemote: false
-

From 5e23382afc834e0f2f11314f1fea7f5dcd141929 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 27 Jan 2023 23:29:22 +0000
Subject: [PATCH 13/27] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 docs/source/operators/deploy-kubernetes.md    | 33 ++++++++++---------
 .../templates/deployment.yaml                 |  8 ++---
 .../templates/kernel-role.yaml                |  2 +-
 .../templates/kubeconfig-configmap.yaml       |  2 +-
 .../templates/role-configmap.yaml             |  2 +-
 5 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/docs/source/operators/deploy-kubernetes.md b/docs/source/operators/deploy-kubernetes.md
index 9fa02e947..2be33bba4 100644
--- a/docs/source/operators/deploy-kubernetes.md
+++ b/docs/source/operators/deploy-kubernetes.md
@@ -520,10 +520,10 @@ can override them with helm's `--set` or `--values` options. Always use `--set`
 | `kip.pullPolicy`                           | Determines whether the Kernel Image Puller will pull kernel images it has previously pulled (`Always`) or only those it hasn't yet pulled (`IfNotPresent`)                                                                                       | `IfNotPresent`                                                                 |
 | `kip.criSocket`                            | The container runtime interface socket, use `/run/containerd/containerd.sock` for containerd installations                                                                                                                                       | `/var/run/docker.sock`                                                         |
 | `kip.defaultContainerRegistry`             | Prefix to use if a registry is not already specified on image name (e.g., elyra/kernel-py:VERSION)                                                                                                                                               | `docker.io`                                                                    |
- | `multicluster.enable` | Launch kernels in a remote cluster. Used for multi-cluster environments. **Must place a kubeconfig file in the `config/` folder of the helm chart**. | `false`
- | `multicluster.configPath` | Path to mount kubeconfig at | `/etc/kube/config`|
- |`multicluster.configFilename` | Filename to kubeconfig file inside `config/` directory of chart | `kubeconfig`|
- | `multicluster.autoconfigureRemote` | Automatically create service account in remote cluster | `false`
+| `multicluster.enable`                      | Launch kernels in a remote cluster. Used for multi-cluster environments. **Must place a kubeconfig file in the `config/` folder of the helm chart**.                                                                                             | `false`                                                                        |
+| `multicluster.configPath`                  | Path to mount kubeconfig at                                                                                                                                                                                                                      | `/etc/kube/config`                                                             |
+| `multicluster.configFilename`              | Filename to kubeconfig file inside `config/` directory of chart                                                                                                                                                                                  | `kubeconfig`                                                                   |
+| `multicluster.autoconfigureRemote`         | Automatically create service account in remote cluster                                                                                                                                                                                           | `false`                                                                        |
 
 ## Uninstalling Enterprise Gateway
 
@@ -962,23 +962,26 @@ The value of the `JUPYTER_GATEWAY_URL` used by the gateway-enabled Notebook serv
 ## Multi-Cluster Environments
 
 ### Overview
+
 With `multicluster.enabled` set to `true`, Enterprise Gateway can be used on multi-cluster environments where the jupyter enterprise gateway pods and kernel pods are launched on separate clusters. To configure this:
 
 1. Ensure your two clusters have interconnceted networks. Pods in the two clusters must be able to communicate with each other over pod IP alone.
-2. Provide a kubeconfig file for use in the `config/` subdirectory of `etc/kubernetes/helm/enterprise-gateway` chart. 
-3. Set `multicluster.enabled` to `true`. 
+1. Provide a kubeconfig file for use in the `config/` subdirectory of `etc/kubernetes/helm/enterprise-gateway` chart.
+1. Set `multicluster.enabled` to `true`.
 
-Enterprise Gateway will now launch kernel pods in whichever cluster you have set to default in your kubeconfig. 
+Enterprise Gateway will now launch kernel pods in whichever cluster you have set to default in your kubeconfig.
 
 ### Resources in Remote Clusters
- For Enterprise Gateway to work accross clusters, Enterprise Gateway must create the following resources in the cluster your kernels will be launched on. 
- - The kernel resource.
- - A service account for the kernel pods (if `multicluster.autoconfigureRemote` is set to `true`).
- - A namespaced role for the namespace where your kernel pods will be launched.
- - A role binding between your namespaced role and your service account. 
-
- The role resource is defined in the `templates/kernel-role.yaml` template of the helm chart. Permissions can be set there. 
- 
+
+For Enterprise Gateway to work accross clusters, Enterprise Gateway must create the following resources in the cluster your kernels will be launched on.
+
+- The kernel resource.
+- A service account for the kernel pods (if `multicluster.autoconfigureRemote` is set to `true`).
+- A namespaced role for the namespace where your kernel pods will be launched.
+- A role binding between your namespaced role and your service account.
+
+The role resource is defined in the `templates/kernel-role.yaml` template of the helm chart. Permissions can be set there.
+
 ## Kubernetes Tips
 
 The following items illustrate some useful commands for navigating Enterprise Gateway within a kubernetes environment.
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
index 9d7b5cff8..172ab11ff 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
@@ -52,7 +52,7 @@ spec:
         volumes:
         {{- if .Values.multicluster.enabled }}
         - name: kubeconfig
-          configMap: 
+          configMap:
             name: {{ .Release.Name }}-configmap
         {{- end}}
         env:
@@ -161,7 +161,7 @@ spec:
           path: "/usr/local/share/jupyter/kernels"
       {{- if .Values.multicluster.enabled }}
       - name: kubeconfig
-        configMap: 
+        configMap:
           name: {{ .Release.Name }}-configmap
       {{- end}}
       {{- if .Values.multicluster.autoconfigureRemote }}
@@ -189,7 +189,7 @@ spec:
           claimName: {{ .Values.kernelspecsPvc.name }}
       {{- if .Values.multicluster.enabled }}
       - name: kubeconfig
-        configMap: 
+        configMap:
           name: {{ .Release.Name }}-configmap
       {{- end}}
       {{- if .Values.multicluster.autoconfigureRemote }}
@@ -217,7 +217,7 @@ spec:
           medium: Memory
       {{- if .Values.multicluster.enabled }}
       - name: kubeconfig
-        configMap: 
+        configMap:
           name: {{ .Release.Name }}-configmap
       {{- end}}
       {{- if .Values.multicluster.autoconfigureRemote }}
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
index 4dc20bb61..fa487e467 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
@@ -22,4 +22,4 @@ rules:
     resources: ["services", "persistentvolumeclaims"]
     verbs: ["list"]
 {{- end }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
index 8f8070207..a78081214 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
@@ -5,4 +5,4 @@ metadata:
   name: {{ .Release.Name }}-configmap
 data:
   {{- (tpl (.Files.Glob "config/*").AsConfig . ) | nindent 2 }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
index 3b53b1f6d..6e3d86b09 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
@@ -8,4 +8,4 @@ data:
   # Allows for Role permissions to be defined via helm chart, not source code
   role.yaml: |
     {{- include "kernel-role" . | indent 4 }}
-{{- end }}
\ No newline at end of file
+{{- end }}

From c9b90cece8254d227c366f16f4722bddc5d8c455 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Mon, 30 Jan 2023 10:43:22 -0700
Subject: [PATCH 14/27] Fix doc requirements

---
 enterprise_gateway/services/utils/envutils.py                 | 4 +++-
 .../operators/scripts/launch_custom_resource.py               | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/enterprise_gateway/services/utils/envutils.py b/enterprise_gateway/services/utils/envutils.py
index b8d97be2c..8e4fc2c0e 100644
--- a/enterprise_gateway/services/utils/envutils.py
+++ b/enterprise_gateway/services/utils/envutils.py
@@ -1,5 +1,7 @@
+"""""Utilities to make checking environment variables easier"""
 import os
 
 
-def is_env_true(env_variable_name: str):
+def is_env_true(env_variable_name: str) -> bool:
+    """If environment variable is set and value is "TRUE" or "true", then return true. Else return false"""
     return bool(os.getenv(env_variable_name, "False").lower() == "true")
diff --git a/etc/kernel-launchers/operators/scripts/launch_custom_resource.py b/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
index e1093b315..0218e43f5 100644
--- a/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
+++ b/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
@@ -58,6 +58,7 @@ def extend_operator_env(op_def: dict, sub_spec: str) -> dict:
 def launch_custom_resource_kernel(
     kernel_id, port_range, response_addr, public_key, spark_context_init_mode
 ):
+    """Launch a custom resource kernel."""
     keywords = dict()
 
     keywords["eg_port_range"] = port_range

From 9ba9ab797ce732bff355c06cc8a15a22766606f3 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 30 Jan 2023 17:45:50 +0000
Subject: [PATCH 15/27] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .../operators/scripts/launch_custom_resource.py                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kernel-launchers/operators/scripts/launch_custom_resource.py b/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
index 279b4b57e..385ccc006 100644
--- a/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
+++ b/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
@@ -59,7 +59,7 @@ def launch_custom_resource_kernel(
     kernel_id, port_range, response_addr, public_key, spark_context_init_mode
 ):
     """Launch a custom resource kernel."""
-    keywords = dict()
+    keywords = {}
     keywords = {}
 
     keywords["eg_port_range"] = port_range

From 0a4c6fb58ad037f280dff6fa7c3497601e0d1846 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Tue, 31 Jan 2023 09:40:43 -0700
Subject: [PATCH 16/27] Change to external cluster naming

---
 docs/source/operators/deploy-kubernetes.md    | 14 +++---
 .../templates/deployment.yaml                 | 50 +++++++++----------
 .../templates/eg-clusterrole.yaml             |  2 +-
 .../templates/kernel-role.yaml                |  2 +-
 .../templates/kubeconfig-configmap.yaml       |  2 +-
 .../templates/role-configmap.yaml             |  2 +-
 6 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/docs/source/operators/deploy-kubernetes.md b/docs/source/operators/deploy-kubernetes.md
index 2be33bba4..8745f5d9d 100644
--- a/docs/source/operators/deploy-kubernetes.md
+++ b/docs/source/operators/deploy-kubernetes.md
@@ -520,10 +520,10 @@ can override them with helm's `--set` or `--values` options. Always use `--set`
 | `kip.pullPolicy`                           | Determines whether the Kernel Image Puller will pull kernel images it has previously pulled (`Always`) or only those it hasn't yet pulled (`IfNotPresent`)                                                                                       | `IfNotPresent`                                                                 |
 | `kip.criSocket`                            | The container runtime interface socket, use `/run/containerd/containerd.sock` for containerd installations                                                                                                                                       | `/var/run/docker.sock`                                                         |
 | `kip.defaultContainerRegistry`             | Prefix to use if a registry is not already specified on image name (e.g., elyra/kernel-py:VERSION)                                                                                                                                               | `docker.io`                                                                    |
-| `multicluster.enable`                      | Launch kernels in a remote cluster. Used for multi-cluster environments. **Must place a kubeconfig file in the `config/` folder of the helm chart**.                                                                                             | `false`                                                                        |
-| `multicluster.configPath`                  | Path to mount kubeconfig at                                                                                                                                                                                                                      | `/etc/kube/config`                                                             |
-| `multicluster.configFilename`              | Filename to kubeconfig file inside `config/` directory of chart                                                                                                                                                                                  | `kubeconfig`                                                                   |
-| `multicluster.autoconfigureRemote`         | Automatically create service account in remote cluster                                                                                                                                                                                           | `false`                                                                        |
+| `externalCluster.enable`                      | Launch kernels in a remote cluster. Used for multi-cluster environments. **Must place a kubeconfig file in the `config/` folder of the helm chart**.                                                                                             | `false`                                                                        |
+| `externalCluster.configPath`                  | Path to mount kubeconfig at                                                                                                                                                                                                                      | `/etc/kube/config`                                                             |
+| `externalCluster.configFilename`              | Filename to kubeconfig file inside `config/` directory of chart                                                                                                                                                                                  | `kubeconfig`                                                                   |
+| `externalCluster.autoconfigureRemote`         | Automatically create service account in remote cluster                                                                                                                                                                                           | `false`                                                                        |
 
 ## Uninstalling Enterprise Gateway
 
@@ -963,11 +963,11 @@ The value of the `JUPYTER_GATEWAY_URL` used by the gateway-enabled Notebook serv
 
 ### Overview
 
-With `multicluster.enabled` set to `true`, Enterprise Gateway can be used on multi-cluster environments where the jupyter enterprise gateway pods and kernel pods are launched on separate clusters. To configure this:
+With `externalCluster.enabled` set to `true`, Enterprise Gateway can be used on multi-cluster environments where the jupyter enterprise gateway pods and kernel pods are launched on separate clusters. To configure this:
 
 1. Ensure your two clusters have interconnceted networks. Pods in the two clusters must be able to communicate with each other over pod IP alone.
 1. Provide a kubeconfig file for use in the `config/` subdirectory of `etc/kubernetes/helm/enterprise-gateway` chart.
-1. Set `multicluster.enabled` to `true`.
+1. Set `externalCluster.enabled` to `true`.
 
 Enterprise Gateway will now launch kernel pods in whichever cluster you have set to default in your kubeconfig.
 
@@ -976,7 +976,7 @@ Enterprise Gateway will now launch kernel pods in whichever cluster you have set
 For Enterprise Gateway to work accross clusters, Enterprise Gateway must create the following resources in the cluster your kernels will be launched on.
 
 - The kernel resource.
-- A service account for the kernel pods (if `multicluster.autoconfigureRemote` is set to `true`).
+- A service account for the kernel pods (if `externalCluster.autoconfigureRemote` is set to `true`).
 - A namespaced role for the namespace where your kernel pods will be launched.
 - A role binding between your namespaced role and your service account.
 
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
index 172ab11ff..ba25141af 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
@@ -44,24 +44,24 @@ spec:
         volumeMounts:
         - name: image-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
-        {{- if .Values.multicluster.enabled }}
+        {{- if .Values.externalCluster.enabled }}
         - name: kubeconfig
           readOnly: true
-          mountPath: {{ .Values.multicluster.configPath }}
+          mountPath: {{ .Values.externalCluster.configPath }}
         {{- end}}
         volumes:
-        {{- if .Values.multicluster.enabled }}
+        {{- if .Values.externalCluster.enabled }}
         - name: kubeconfig
           configMap:
             name: {{ .Release.Name }}-configmap
         {{- end}}
         env:
         # Set environment variables to point k8s client to kubeconfig
-        {{- if .Values.multicluster.enabled }}
+        {{- if .Values.externalCluster.enabled }}
           - name: EG_USE_REMOTE_CLUSTER
             value: "True"
           - name: EG_REMOTE_CLUSTER_KUBECONFIG_PATH
-            value: {{ .Values.multicluster.configPath }}{{ .Values.multicluster.configFilename }}
+            value: {{ .Values.externalCluster.configPath }}{{ .Values.externalCluster.configFilename }}
         {{- end }}
   {{- end }}
       containers:
@@ -98,18 +98,18 @@ spec:
         - name: EG_DEFAULT_KERNEL_NAME
           value: {{ .Values.kernel.defaultKernelName }}
         # Set environment variables to point k8s client to kubeconfig
-        {{- if .Values.multicluster.enabled }}
+        {{- if .Values.externalCluster.enabled }}
         - name: EG_USE_REMOTE_CLUSTER
           value: "True"
         - name: EG_REMOTE_CLUSTER_KUBECONFIG_PATH
-          value: {{ .Values.multicluster.configPath }}{{ .Values.multicluster.configFilename }}
+          value: {{ .Values.externalCluster.configPath }}{{ .Values.externalCluster.configFilename }}
         - name: EG_DEFAULT_KERNEL_SERVICE_ACCOUNT_NAME
           value: kernel-svcaccount
-        {{- if .Values.multicluster.context }}
+        {{- if .Values.externalCluster.context }}
         - name: EG_REMOTE_CLUSTER_CONTEXT
-          value: {{ .Values.multicluster.context }}
+          value: {{ .Values.externalCluster.context }}
         {{- end}}
-        {{- if .Values.multicluster.autoconfigureRemote }}
+        {{- if .Values.externalCluster.autoconfigureRemote }}
         - name: EG_CREATE_REMOTE_SVC_ACCOUNT
           value: "True"
         - name: EG_REMOTE_CLUSTER_ROLE_PATH
@@ -144,12 +144,12 @@ spec:
         volumeMounts:
         - name: nfs-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
-        {{- if .Values.multicluster.enabled }}
+        {{- if .Values.externalCluster.enabled }}
         - name: kubeconfig
           readOnly: true
-          mountPath: {{ .Values.multicluster.configPath }}
+          mountPath: {{ .Values.externalCluster.configPath }}
         {{- end}}
-        {{- if .Values.multicluster.autoconfigureRemote }}
+        {{- if .Values.externalCluster.autoconfigureRemote }}
         - name: role
           readOnly: true
           mountPath: "/etc/config/role"
@@ -159,12 +159,12 @@ spec:
         nfs:
           server: {{ .Values.nfs.internalServerIPAddress }}
           path: "/usr/local/share/jupyter/kernels"
-      {{- if .Values.multicluster.enabled }}
+      {{- if .Values.externalCluster.enabled }}
       - name: kubeconfig
         configMap:
           name: {{ .Release.Name }}-configmap
       {{- end}}
-      {{- if .Values.multicluster.autoconfigureRemote }}
+      {{- if .Values.externalCluster.autoconfigureRemote }}
       - name: role
         configMap:
           name: {{ .Release.name }}-configmap-role
@@ -173,12 +173,12 @@ spec:
         volumeMounts:
         - name: pvc-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
-        {{- if .Values.multicluster.enabled }}
+        {{- if .Values.externalCluster.enabled }}
         - name: kubeconfig
           readOnly: true
-          mountPath: {{ .Values.multicluster.configPath }}
+          mountPath: {{ .Values.externalCluster.configPath }}
         {{- end}}
-        {{- if .Values.multicluster.autoconfigureRemote }}
+        {{- if .Values.externalCluster.autoconfigureRemote }}
         - name: role
           readOnly: true
           mountPath: "/etc/config/role"
@@ -187,12 +187,12 @@ spec:
       - name: pvc-kernelspecs
         persistentVolumeClaim:
           claimName: {{ .Values.kernelspecsPvc.name }}
-      {{- if .Values.multicluster.enabled }}
+      {{- if .Values.externalCluster.enabled }}
       - name: kubeconfig
         configMap:
           name: {{ .Release.Name }}-configmap
       {{- end}}
-      {{- if .Values.multicluster.autoconfigureRemote }}
+      {{- if .Values.externalCluster.autoconfigureRemote }}
       - name: role
         configMap:
           name: {{ .Release.name}}-configmap-role
@@ -201,12 +201,12 @@ spec:
         volumeMounts:
         - name: image-kernelspecs
           mountPath: "/usr/local/share/jupyter/kernels"
-        {{- if .Values.multicluster.enabled }}
+        {{- if .Values.externalCluster.enabled }}
         - name: kubeconfig
           readOnly: true
-          mountPath: {{ .Values.multicluster.configPath }}
+          mountPath: {{ .Values.externalCluster.configPath }}
         {{- end}}
-        {{- if .Values.multicluster.autoconfigureRemote }}
+        {{- if .Values.externalCluster.autoconfigureRemote }}
         - name: role
           readOnly: true
           mountPath: "/etc/config/role"
@@ -215,12 +215,12 @@ spec:
       - name: image-kernelspecs
         emptyDir:
           medium: Memory
-      {{- if .Values.multicluster.enabled }}
+      {{- if .Values.externalCluster.enabled }}
       - name: kubeconfig
         configMap:
           name: {{ .Release.Name }}-configmap
       {{- end}}
-      {{- if .Values.multicluster.autoconfigureRemote }}
+      {{- if .Values.externalCluster.autoconfigureRemote }}
       - name: role
         configMap:
           name: {{ .Release.Name }}-configmap-role
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
index 55c703441..93ae5c1ea 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
@@ -24,7 +24,7 @@ rules:
     resources: ["sparkapplications", "sparkapplications/status", "scheduledsparkapplications", "scheduledsparkapplications/status"]
     verbs: ["get", "watch", "list", "create", "delete"]
   # If multicluster used, need to be able to read and forward service account
-  {{- if .Values.multicluster.enabled }}
+  {{- if .Values.externalCluster.enabled }}
   - apiGroups: ["rbac.authorization.k8s.io"]
     resources: ["clusterroles"]
     verbs: ["get", "create"]
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
index fa487e467..6a58eab31 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
@@ -1,5 +1,5 @@
 {{- define "kernel-role" }}
-{{- if .Values.multicluster.autoconfigureRemote }}
+{{- if .Values.externalCluster.autoconfigureRemote }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
index a78081214..cc011f7b6 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/kubeconfig-configmap.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.multicluster.enabled }}
+{{- if .Values.externalCluster.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
index 6e3d86b09..e8d008384 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.multicluster.autoconfigureRemote }}
+{{- if .Values.externalCluster.autoconfigureRemote }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

From d8b55b66c6629c369e121d0ff640e2088ae52679 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 31 Jan 2023 16:41:48 +0000
Subject: [PATCH 17/27] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 docs/source/operators/deploy-kubernetes.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/source/operators/deploy-kubernetes.md b/docs/source/operators/deploy-kubernetes.md
index 8745f5d9d..2f75eff78 100644
--- a/docs/source/operators/deploy-kubernetes.md
+++ b/docs/source/operators/deploy-kubernetes.md
@@ -520,10 +520,10 @@ can override them with helm's `--set` or `--values` options. Always use `--set`
 | `kip.pullPolicy`                           | Determines whether the Kernel Image Puller will pull kernel images it has previously pulled (`Always`) or only those it hasn't yet pulled (`IfNotPresent`)                                                                                       | `IfNotPresent`                                                                 |
 | `kip.criSocket`                            | The container runtime interface socket, use `/run/containerd/containerd.sock` for containerd installations                                                                                                                                       | `/var/run/docker.sock`                                                         |
 | `kip.defaultContainerRegistry`             | Prefix to use if a registry is not already specified on image name (e.g., elyra/kernel-py:VERSION)                                                                                                                                               | `docker.io`                                                                    |
-| `externalCluster.enable`                      | Launch kernels in a remote cluster. Used for multi-cluster environments. **Must place a kubeconfig file in the `config/` folder of the helm chart**.                                                                                             | `false`                                                                        |
-| `externalCluster.configPath`                  | Path to mount kubeconfig at                                                                                                                                                                                                                      | `/etc/kube/config`                                                             |
-| `externalCluster.configFilename`              | Filename to kubeconfig file inside `config/` directory of chart                                                                                                                                                                                  | `kubeconfig`                                                                   |
-| `externalCluster.autoconfigureRemote`         | Automatically create service account in remote cluster                                                                                                                                                                                           | `false`                                                                        |
+| `externalCluster.enable`                   | Launch kernels in a remote cluster. Used for multi-cluster environments. **Must place a kubeconfig file in the `config/` folder of the helm chart**.                                                                                             | `false`                                                                        |
+| `externalCluster.configPath`               | Path to mount kubeconfig at                                                                                                                                                                                                                      | `/etc/kube/config`                                                             |
+| `externalCluster.configFilename`           | Filename to kubeconfig file inside `config/` directory of chart                                                                                                                                                                                  | `kubeconfig`                                                                   |
+| `externalCluster.autoconfigureRemote`      | Automatically create service account in remote cluster                                                                                                                                                                                           | `false`                                                                        |
 
 ## Uninstalling Enterprise Gateway
 

From 94ab66e7cf8d5e66aee1eee60e571ab1d99285eb Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Tue, 31 Jan 2023 09:49:33 -0700
Subject: [PATCH 18/27] Chart lint fixes

---
 .../helm/enterprise-gateway/templates/eg-clusterrole.yaml       | 2 +-
 etc/kubernetes/helm/enterprise-gateway/values.yaml              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
index 93ae5c1ea..4e2c20051 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/eg-clusterrole.yaml
@@ -23,7 +23,7 @@ rules:
   - apiGroups: ["sparkoperator.k8s.io"]
     resources: ["sparkapplications", "sparkapplications/status", "scheduledsparkapplications", "scheduledsparkapplications/status"]
     verbs: ["get", "watch", "list", "create", "delete"]
-  # If multicluster used, need to be able to read and forward service account
+  # If externalCluster used, need to be able to read and forward service account
   {{- if .Values.externalCluster.enabled }}
   - apiGroups: ["rbac.authorization.k8s.io"]
     resources: ["clusterroles"]
diff --git a/etc/kubernetes/helm/enterprise-gateway/values.yaml b/etc/kubernetes/helm/enterprise-gateway/values.yaml
index 811783bca..e318afb3e 100644
--- a/etc/kubernetes/helm/enterprise-gateway/values.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/values.yaml
@@ -180,7 +180,7 @@ kip:
   #     cpu: 1
   #     memory: 1Gi
 
-multicluster:
+externalCluster:
   # Set to enable to point k8s client to remote cluster
   enabled: false
   # Mount config at this path inside pod

From 4fbe8b7889363eaab553227f7299caef3a6d7d0e Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Fri, 14 Apr 2023 14:49:32 -0600
Subject: [PATCH 19/27] Update docs/source/operators/deploy-kubernetes.md

Co-authored-by: Kevin Bates <kbates4@gmail.com>
---
 docs/source/operators/deploy-kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/operators/deploy-kubernetes.md b/docs/source/operators/deploy-kubernetes.md
index 2f75eff78..ccbd624a3 100644
--- a/docs/source/operators/deploy-kubernetes.md
+++ b/docs/source/operators/deploy-kubernetes.md
@@ -523,7 +523,7 @@ can override them with helm's `--set` or `--values` options. Always use `--set`
 | `externalCluster.enable`                   | Launch kernels in a remote cluster. Used for multi-cluster environments. **Must place a kubeconfig file in the `config/` folder of the helm chart**.                                                                                             | `false`                                                                        |
 | `externalCluster.configPath`               | Path to mount kubeconfig at                                                                                                                                                                                                                      | `/etc/kube/config`                                                             |
 | `externalCluster.configFilename`           | Filename to kubeconfig file inside `config/` directory of chart                                                                                                                                                                                  | `kubeconfig`                                                                   |
-| `externalCluster.autoconfigureRemote`      | Automatically create service account in remote cluster                                                                                                                                                                                           | `false`                                                                        |
+| `externalCluster.autoConfigureRemote`      | Automatically create service account in remote cluster                                                                                                                                                                                           | `false`                                                                        |
 
 ## Uninstalling Enterprise Gateway
 

From 51256b979c131b702c1f27243cddefea2300ec38 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Fri, 14 Apr 2023 14:49:42 -0600
Subject: [PATCH 20/27] Update docs/source/operators/deploy-kubernetes.md

Co-authored-by: Kevin Bates <kbates4@gmail.com>
---
 docs/source/operators/deploy-kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/operators/deploy-kubernetes.md b/docs/source/operators/deploy-kubernetes.md
index ccbd624a3..e0fdfbdc0 100644
--- a/docs/source/operators/deploy-kubernetes.md
+++ b/docs/source/operators/deploy-kubernetes.md
@@ -976,7 +976,7 @@ Enterprise Gateway will now launch kernel pods in whichever cluster you have set
 For Enterprise Gateway to work accross clusters, Enterprise Gateway must create the following resources in the cluster your kernels will be launched on.
 
 - The kernel resource.
-- A service account for the kernel pods (if `externalCluster.autoconfigureRemote` is set to `true`).
+- A service account for the kernel pods (if `externalCluster.autoConfigureRemote` is set to `true`).
 - A namespaced role for the namespace where your kernel pods will be launched.
 - A role binding between your namespaced role and your service account.
 

From 8107dac117cd02891b6312825364f4a606db0ccb Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Fri, 14 Apr 2023 14:51:13 -0600
Subject: [PATCH 21/27] Update
 enterprise_gateway/services/processproxies/crd.py

Co-authored-by: Kevin Bates <kbates4@gmail.com>
---
 enterprise_gateway/services/processproxies/crd.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/enterprise_gateway/services/processproxies/crd.py b/enterprise_gateway/services/processproxies/crd.py
index a8ff3426a..b7394fea3 100644
--- a/enterprise_gateway/services/processproxies/crd.py
+++ b/enterprise_gateway/services/processproxies/crd.py
@@ -39,7 +39,7 @@ async def launch_process(
         kwargs["env"]["KERNEL_CRD_VERSION"] = self.version
         kwargs["env"]["KERNEL_CRD_PLURAL"] = self.plural
 
-        use_remote_cluster = os.getenv("EG_USE_REMOTE_CLUSTER")
+        use_remote_cluster = is_env_true("EG_USE_REMOTE_CLUSTER")
         if use_remote_cluster:
             kwargs["env"]["EG_USE_REMOTE_CLUSTER"] = 'true'
             kwargs["env"]["EG_REMOTE_CLUSTER_KUBECONFIG_PATH"] = os.getenv(

From 26cba1ab805f777883188f6a83ff2010e3164fee Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Fri, 14 Apr 2023 14:52:26 -0600
Subject: [PATCH 22/27] Apply suggestions from code review

Co-authored-by: Kevin Bates <kbates4@gmail.com>
---
 enterprise_gateway/services/processproxies/k8s.py      |  2 +-
 enterprise_gateway/services/utils/envutils.py          |  2 +-
 .../helm/enterprise-gateway/templates/deployment.yaml  | 10 +++++-----
 .../helm/enterprise-gateway/templates/kernel-role.yaml |  2 +-
 .../enterprise-gateway/templates/role-configmap.yaml   |  2 +-
 etc/kubernetes/helm/enterprise-gateway/values.yaml     |  2 +-
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index 6b612ddf6..a3c77752a 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -287,7 +287,7 @@ def _create_kernel_namespace(self, service_account_name: str) -> str:
 
             # If remote cluster is being used, service account may not be present, create before role binding
             # If creating service account is disabled, operator must manually create svc account
-            if is_env_true('EG_USE_REMOTE_CLUSTER') and os.getenv('EG_CREATE_REMOTE_SVC_ACCOUNT'):
+            if is_env_true('EG_USE_REMOTE_CLUSTER') and is_env_true('EG_CREATE_REMOTE_SVC_ACCOUNT'):
                 self._create_service_account_if_not_exists(
                     namespace=namespace, service_account_name=service_account_name
                 )
diff --git a/enterprise_gateway/services/utils/envutils.py b/enterprise_gateway/services/utils/envutils.py
index 8e4fc2c0e..a8135edd1 100644
--- a/enterprise_gateway/services/utils/envutils.py
+++ b/enterprise_gateway/services/utils/envutils.py
@@ -3,5 +3,5 @@
 
 
 def is_env_true(env_variable_name: str) -> bool:
-    """If environment variable is set and value is "TRUE" or "true", then return true. Else return false"""
+    """If environment variable is set and value is case-insensitively "true", then return true. Else return false"""
     return bool(os.getenv(env_variable_name, "False").lower() == "true")
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
index ba25141af..9c5a64f7e 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
@@ -149,7 +149,7 @@ spec:
           readOnly: true
           mountPath: {{ .Values.externalCluster.configPath }}
         {{- end}}
-        {{- if .Values.externalCluster.autoconfigureRemote }}
+        {{- if .Values.externalCluster.autoConfigureRemote }}
         - name: role
           readOnly: true
           mountPath: "/etc/config/role"
@@ -164,7 +164,7 @@ spec:
         configMap:
           name: {{ .Release.Name }}-configmap
       {{- end}}
-      {{- if .Values.externalCluster.autoconfigureRemote }}
+      {{- if .Values.externalCluster.autoConfigureRemote }}
       - name: role
         configMap:
           name: {{ .Release.name }}-configmap-role
@@ -192,7 +192,7 @@ spec:
         configMap:
           name: {{ .Release.Name }}-configmap
       {{- end}}
-      {{- if .Values.externalCluster.autoconfigureRemote }}
+      {{- if .Values.externalCluster.autoConfigureRemote }}
       - name: role
         configMap:
           name: {{ .Release.name}}-configmap-role
@@ -206,7 +206,7 @@ spec:
           readOnly: true
           mountPath: {{ .Values.externalCluster.configPath }}
         {{- end}}
-        {{- if .Values.externalCluster.autoconfigureRemote }}
+        {{- if .Values.externalCluster.autoConfigureRemote }}
         - name: role
           readOnly: true
           mountPath: "/etc/config/role"
@@ -220,7 +220,7 @@ spec:
         configMap:
           name: {{ .Release.Name }}-configmap
       {{- end}}
-      {{- if .Values.externalCluster.autoconfigureRemote }}
+      {{- if .Values.externalCluster.autoConfigureRemote }}
       - name: role
         configMap:
           name: {{ .Release.Name }}-configmap-role
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
index 6a58eab31..da975bb5f 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/kernel-role.yaml
@@ -1,5 +1,5 @@
 {{- define "kernel-role" }}
-{{- if .Values.externalCluster.autoconfigureRemote }}
+{{- if .Values.externalCluster.autoConfigureRemote }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
index e8d008384..275203367 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/role-configmap.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.externalCluster.autoconfigureRemote }}
+{{- if .Values.externalCluster.autoConfigureRemote }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
diff --git a/etc/kubernetes/helm/enterprise-gateway/values.yaml b/etc/kubernetes/helm/enterprise-gateway/values.yaml
index e318afb3e..f2ae16893 100644
--- a/etc/kubernetes/helm/enterprise-gateway/values.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/values.yaml
@@ -188,4 +188,4 @@ externalCluster:
   # Filename of kubeconfig
   configFilename: kubeconfig
   # Automatically create service account and role in remote cluster
-  autoconfigureRemote: false
+  autoConfigureRemote: false

From 55065ca837a4f309e0355d56e763148df8716ff8 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Fri, 14 Apr 2023 14:52:43 -0600
Subject: [PATCH 23/27] Update docs/source/operators/deploy-kubernetes.md

Co-authored-by: Kevin Bates <kbates4@gmail.com>
---
 docs/source/operators/deploy-kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/operators/deploy-kubernetes.md b/docs/source/operators/deploy-kubernetes.md
index e0fdfbdc0..3b5913956 100644
--- a/docs/source/operators/deploy-kubernetes.md
+++ b/docs/source/operators/deploy-kubernetes.md
@@ -973,7 +973,7 @@ Enterprise Gateway will now launch kernel pods in whichever cluster you have set
 
 ### Resources in Remote Clusters
 
-For Enterprise Gateway to work accross clusters, Enterprise Gateway must create the following resources in the cluster your kernels will be launched on.
+For Enterprise Gateway to work across clusters, Enterprise Gateway must create the following resources in the cluster your kernels will be launched on.
 
 - The kernel resource.
 - A service account for the kernel pods (if `externalCluster.autoConfigureRemote` is set to `true`).

From 3282e143fc65fd96e2a3f9efc77237e900213009 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Fri, 14 Apr 2023 15:00:24 -0600
Subject: [PATCH 24/27] Update
 etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml

Co-authored-by: Kevin Bates <kbates4@gmail.com>
---
 .../helm/enterprise-gateway/templates/deployment.yaml           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
index 9c5a64f7e..91310c4cd 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
@@ -178,7 +178,7 @@ spec:
           readOnly: true
           mountPath: {{ .Values.externalCluster.configPath }}
         {{- end}}
-        {{- if .Values.externalCluster.autoconfigureRemote }}
+        {{- if .Values.externalCluster.autoConfigureRemote }}
         - name: role
           readOnly: true
           mountPath: "/etc/config/role"

From 047538e2052ef455a812ba886f5aff47b185edae Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Fri, 14 Apr 2023 15:03:39 -0600
Subject: [PATCH 25/27] progress

---
 enterprise_gateway/services/external/__init__.py   |  0
 enterprise_gateway/services/processproxies/crd.py  |  9 +--------
 enterprise_gateway/services/processproxies/k8s.py  | 10 ++++++++--
 .../{external => processproxies}/k8s_client.py     |  2 +-
 .../k8s_client_factory.py                          |  8 ++------
 etc/docker/enterprise-gateway/Dockerfile           | 14 ++++++++++++++
 .../kubernetes/scripts/launch_kubernetes.py        |  2 +-
 .../operators/scripts/launch_custom_resource.py    |  2 +-
 .../enterprise-gateway/templates/deployment.yaml   | 10 ++++++++--
 etc/kubernetes/helm/enterprise-gateway/values.yaml |  4 ++++
 10 files changed, 40 insertions(+), 21 deletions(-)
 delete mode 100644 enterprise_gateway/services/external/__init__.py
 rename enterprise_gateway/services/{external => processproxies}/k8s_client.py (66%)
 rename enterprise_gateway/services/{external => processproxies}/k8s_client_factory.py (85%)

diff --git a/enterprise_gateway/services/external/__init__.py b/enterprise_gateway/services/external/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/enterprise_gateway/services/processproxies/crd.py b/enterprise_gateway/services/processproxies/crd.py
index a8ff3426a..285ec93ba 100644
--- a/enterprise_gateway/services/processproxies/crd.py
+++ b/enterprise_gateway/services/processproxies/crd.py
@@ -9,7 +9,7 @@
 
 from kubernetes import client
 
-from ..external.k8s_client import kubernetes_client
+from enterprise_gateway.services.processproxies.k8s_client import kubernetes_client
 from ..kernels.remotemanager import RemoteKernelManager
 from .k8s import KubernetesProcessProxy
 
@@ -39,13 +39,6 @@ async def launch_process(
         kwargs["env"]["KERNEL_CRD_VERSION"] = self.version
         kwargs["env"]["KERNEL_CRD_PLURAL"] = self.plural
 
-        use_remote_cluster = os.getenv("EG_USE_REMOTE_CLUSTER")
-        if use_remote_cluster:
-            kwargs["env"]["EG_USE_REMOTE_CLUSTER"] = 'true'
-            kwargs["env"]["EG_REMOTE_CLUSTER_KUBECONFIG_PATH"] = os.getenv(
-                "EG_REMOTE_CLUSTER_KUBECONFIG_PATH"
-            )
-
         await super().launch_process(kernel_cmd, **kwargs)
         return self
 
diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index 6b612ddf6..0b71b36ff 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -14,7 +14,7 @@
 from kubernetes import client
 from kubernetes.utils.create_from_yaml import create_from_yaml_single_item
 
-from ..external.k8s_client import kubernetes_client
+from enterprise_gateway.services.processproxies.k8s_client import kubernetes_client
 from ..kernels.remotemanager import RemoteKernelManager
 from ..sessions.kernelsessionmanager import KernelSessionManager
 from ..utils.envutils import is_env_true
@@ -58,6 +58,12 @@ async def launch_process(
     ) -> "KubernetesProcessProxy":
         """Launches the specified process within a Kubernetes environment."""
         # Set env before superclass call, so we can see these in the debug output
+        use_remote_cluster = os.getenv("EG_USE_REMOTE_CLUSTER")
+        if use_remote_cluster:
+            kwargs["env"]["EG_USE_REMOTE_CLUSTER"] = 'true'
+            kwargs["env"]["EG_REMOTE_CLUSTER_KUBECONFIG_PATH"] = os.getenv(
+                "EG_REMOTE_CLUSTER_KUBECONFIG_PATH"
+            )
 
         # Kubernetes relies on internal env variables to determine its configuration.  When
         # running within a K8s cluster, these start with KUBERNETES_SERVICE, otherwise look
@@ -237,7 +243,7 @@ def _determine_kernel_namespace(self, **kwargs: dict[str, Any] | None) -> str:
 
         # If KERNEL_NAMESPACE was provided, then we assume it already exists.  If not provided, then we'll
         # create the namespace and record that we'll want to delete it as well.
-        namespace = kwargs["env"].get("KERNEL_NAMESPACE")
+        namespace = os.environ.get("KERNEL_NAMESPACE")
         if namespace is None:
             # check if share gateway namespace is configured...
             if share_gateway_namespace:  # if so, set to EG namespace
diff --git a/enterprise_gateway/services/external/k8s_client.py b/enterprise_gateway/services/processproxies/k8s_client.py
similarity index 66%
rename from enterprise_gateway/services/external/k8s_client.py
rename to enterprise_gateway/services/processproxies/k8s_client.py
index 7d93188ef..369aebd65 100644
--- a/enterprise_gateway/services/external/k8s_client.py
+++ b/enterprise_gateway/services/processproxies/k8s_client.py
@@ -1,5 +1,5 @@
 """Instantiates a static global factory and a single atomic client"""
-from enterprise_gateway.services.external.k8s_client_factory import KubernetesClientFactory
+from enterprise_gateway.services.processproxies.k8s_client_factory import KubernetesClientFactory
 
 KUBERNETES_CLIENT_FACTORY = KubernetesClientFactory()
 kubernetes_client = KUBERNETES_CLIENT_FACTORY.get_kubernetes_client()
diff --git a/enterprise_gateway/services/external/k8s_client_factory.py b/enterprise_gateway/services/processproxies/k8s_client_factory.py
similarity index 85%
rename from enterprise_gateway/services/external/k8s_client_factory.py
rename to enterprise_gateway/services/processproxies/k8s_client_factory.py
index 46b79130d..9b2182e01 100644
--- a/enterprise_gateway/services/external/k8s_client_factory.py
+++ b/enterprise_gateway/services/processproxies/k8s_client_factory.py
@@ -4,17 +4,13 @@
 from kubernetes import client, config
 from traitlets.config import SingletonConfigurable
 
-from ..utils.envutils import is_env_true
+from enterprise_gateway.services.utils.envutils import is_env_true
 
 
 class KubernetesClientFactory(SingletonConfigurable):
     """Manages kubernetes client creation from environment variables"""
 
-    def __init__(self) -> None:
-        """Maintain a single configuration object and populate based on environment"""
-        super().__init__()
-
-    def get_kubernetes_client(self, get_remote_client: bool = True) -> client.ApiClient:
+    def get_kubernetes_client(self, get_remote_client: bool = False) -> client.ApiClient:
         """Get kubernetes api client with appropriate configuration
 
         Args:
diff --git a/etc/docker/enterprise-gateway/Dockerfile b/etc/docker/enterprise-gateway/Dockerfile
index 2adb73ea6..b31bee399 100644
--- a/etc/docker/enterprise-gateway/Dockerfile
+++ b/etc/docker/enterprise-gateway/Dockerfile
@@ -25,6 +25,20 @@ RUN apt update && apt install -yq curl openjdk-8-jdk
 ENV JAVA_HOME /usr/lib/jvm/java
 RUN ln -s $(readlink -f /usr/bin/javac | sed "s:/bin/javac::") ${JAVA_HOME}
 
+RUN curl https://apt.releases.teleport.dev/gpg \
+-o /usr/share/keyrings/teleport-archive-keyring.asc
+
+RUN source /etc/os-release
+RUN echo "deb [signed-by=/usr/share/keyrings/teleport-archive-keyring.asc] \
+https://apt.releases.teleport.dev/ubuntu jammy stable/v11" \
+| tee /etc/apt/sources.list.d/teleport.list > /dev/null
+
+RUN apt-get update && apt-get install teleport
+
+# Download and install kubectl
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+RUN sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
 # Download and install Spark
 RUN curl -s https://archive.apache.org/dist/spark/spark-${SPARK_VER}/spark-${SPARK_VER}-bin-hadoop2.7.tgz | \
     tar -xz -C /opt && \
diff --git a/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py b/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py
index 51dd3a423..ea06703fe 100644
--- a/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py
+++ b/etc/kernel-launchers/kubernetes/scripts/launch_kubernetes.py
@@ -11,7 +11,7 @@
 from kubernetes import client
 from kubernetes.client.rest import ApiException
 
-from enterprise_gateway.services.external.k8s_client import kubernetes_client
+from enterprise_gateway.services.processproxies.k8s_client import kubernetes_client
 
 urllib3.disable_warnings()
 
diff --git a/etc/kernel-launchers/operators/scripts/launch_custom_resource.py b/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
index 385ccc006..096a83bb5 100644
--- a/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
+++ b/etc/kernel-launchers/operators/scripts/launch_custom_resource.py
@@ -9,7 +9,7 @@
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 from kubernetes import client
 
-from enterprise_gateway.services.external.k8s_client import kubernetes_client
+from enterprise_gateway.services.processproxies.k8s_client import kubernetes_client
 
 urllib3.disable_warnings()
 
diff --git a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
index ba25141af..dd9b005fd 100644
--- a/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/templates/deployment.yaml
@@ -97,14 +97,20 @@ spec:
           value: {{ toJson .Values.kernel.allowedKernels | squote }}
         - name: EG_DEFAULT_KERNEL_NAME
           value: {{ .Values.kernel.defaultKernelName }}
+          {{- if .Values.kernel.serviceAccountName }}
+        - name: EG_DEFAULT_KERNEL_SERVICE_ACCOUNT_NAME
+          value: {{ .Values.kernel.serviceAccountName }}
+          {{- end }}
+          {{- if .Values.kernel.namespace }}
+        - name: KERNEL_NAMESPACE
+          value: {{ .Values.kernel.namespace }}
+          {{- end }}
         # Set environment variables to point k8s client to kubeconfig
         {{- if .Values.externalCluster.enabled }}
         - name: EG_USE_REMOTE_CLUSTER
           value: "True"
         - name: EG_REMOTE_CLUSTER_KUBECONFIG_PATH
           value: {{ .Values.externalCluster.configPath }}{{ .Values.externalCluster.configFilename }}
-        - name: EG_DEFAULT_KERNEL_SERVICE_ACCOUNT_NAME
-          value: kernel-svcaccount
         {{- if .Values.externalCluster.context }}
         - name: EG_REMOTE_CLUSTER_CONTEXT
           value: {{ .Values.externalCluster.context }}
diff --git a/etc/kubernetes/helm/enterprise-gateway/values.yaml b/etc/kubernetes/helm/enterprise-gateway/values.yaml
index e318afb3e..fbb1ba4ae 100644
--- a/etc/kubernetes/helm/enterprise-gateway/values.yaml
+++ b/etc/kubernetes/helm/enterprise-gateway/values.yaml
@@ -85,6 +85,10 @@ authToken:
 kernel:
   # Kernel cluster role created by this chart.
   clusterRole: kernel-controller
+  # Service Account to Use (optional)
+  serviceAccountName:
+  # Namespace to launch in (optional)
+  namespace:
   # Will start kernels in the same namespace as EG if True.
   shareGatewayNamespace: false
   # Timeout for kernel launching in seconds.

From 7c06f2b7cafe26edaaadfc1767067fd2c4ad50f3 Mon Sep 17 00:00:00 2001
From: Shrinjay Mukherjee <62576642+Shrinjay@users.noreply.github.com>
Date: Fri, 14 Apr 2023 15:16:24 -0600
Subject: [PATCH 26/27] throw warning

---
 .../services/processproxies/k8s_client_factory.py     | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/enterprise_gateway/services/processproxies/k8s_client_factory.py b/enterprise_gateway/services/processproxies/k8s_client_factory.py
index 9b2182e01..0b29aa7ce 100644
--- a/enterprise_gateway/services/processproxies/k8s_client_factory.py
+++ b/enterprise_gateway/services/processproxies/k8s_client_factory.py
@@ -10,19 +10,15 @@
 class KubernetesClientFactory(SingletonConfigurable):
     """Manages kubernetes client creation from environment variables"""
 
-    def get_kubernetes_client(self, get_remote_client: bool = False) -> client.ApiClient:
+    def get_kubernetes_client(self) -> client.ApiClient:
         """Get kubernetes api client with appropriate configuration
-
-        Args:
-            get_remote_client (bool): Return a client for the remote cluster if configured. Else, return incluster config. Defaults to True.
-
         Returns:
             ApiClient: Kubernetes API client for appropriate cluster
         """
         kubernetes_config: client.Configuration = client.Configuration()
         if os.getenv("KUBERNETES_SERVICE_HOST"):
             # Running inside cluster
-            if is_env_true('EG_USE_REMOTE_CLUSTER') and get_remote_client:
+            if is_env_true('EG_USE_REMOTE_CLUSTER') and not is_env_true('EG_SHARED_NAMESPACE'):
                 kubeconfig_path = os.getenv(
                     'EG_REMOTE_CLUSTER_KUBECONFIG_PATH', '/etc/kube/config/kubeconfig'
                 )
@@ -33,6 +29,9 @@ def get_kubernetes_client(self, get_remote_client: bool = False) -> client.ApiCl
                     context=context,
                 )
             else:
+                if is_env_true('EG_USE_REMOTE_CLUSTER'):
+                    self.log.warning(f"Cannot use EG_USE_REMOTE_CLUSTER and EG_SHARED_NAMESPACE at the same time. Using local cluster....")
+
                 config.load_incluster_config(client_configuration=kubernetes_config)
         else:
             config.load_kube_config(client_configuration=kubernetes_config)

From 85543437ffc111c91840e4dc13303ac487ded94e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Fri, 14 Apr 2023 21:21:09 +0000
Subject: [PATCH 27/27] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 docs/source/operators/deploy-kubernetes.md                    | 4 ++--
 enterprise_gateway/services/processproxies/crd.py             | 2 +-
 enterprise_gateway/services/processproxies/k8s.py             | 1 +
 .../services/processproxies/k8s_client_factory.py             | 4 +++-
 4 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/docs/source/operators/deploy-kubernetes.md b/docs/source/operators/deploy-kubernetes.md
index 7f9ca98ac..97ce1e003 100644
--- a/docs/source/operators/deploy-kubernetes.md
+++ b/docs/source/operators/deploy-kubernetes.md
@@ -522,11 +522,11 @@ can override them with helm's `--set` or `--values` options. Always use `--set`
 | `kip.defaultContainerRegistry`             | Prefix to use if a registry is not already specified on image name (e.g., elyra/kernel-py:VERSION)                                                                                                                                               | `docker.io`                                                                    |
 | `kip.fetcher`                              | fetcher to fetch image names, defaults to KernelSpecsFetcher                                                                                                                                                                                     | `KernelSpecsFetcher`                                                           |
 | `kip.images`                               | if StaticListFetcher is used KIP_IMAGES defines the list of images pullers will fetch                                                                                                                                                            | `[]`                                                                           |
-| `kip.internalFetcher `                     | if CombinedImagesFetcher is used KIP_INTERNAL_FETCHERS defines the fetchers that get used internally 
+| `kip.internalFetcher `                     | if CombinedImagesFetcher is used KIP_INTERNAL_FETCHERS defines the fetchers that get used internally                                                                                                                                             |                                                                                |
 | `externalCluster.enable`                   | Launch kernels in a remote cluster. Used for multi-cluster environments. **Must place a kubeconfig file in the `config/` folder of the helm chart**.                                                                                             | `false`                                                                        |
 | `externalCluster.configPath`               | Path to mount kubeconfig at                                                                                                                                                                                                                      | `/etc/kube/config`                                                             |
 | `externalCluster.configFilename`           | Filename to kubeconfig file inside `config/` directory of chart                                                                                                                                                                                  | `kubeconfig`                                                                   |
-| `externalCluster.autoConfigureRemote`      | Automatically create service account in remote cluster                    
+| `externalCluster.autoConfigureRemote`      | Automatically create service account in remote cluster                                                                                                                                                                                           |                                                                                |
 
 ## Uninstalling Enterprise Gateway
 
diff --git a/enterprise_gateway/services/processproxies/crd.py b/enterprise_gateway/services/processproxies/crd.py
index da7d5e17d..ba3fc2eb2 100644
--- a/enterprise_gateway/services/processproxies/crd.py
+++ b/enterprise_gateway/services/processproxies/crd.py
@@ -4,7 +4,6 @@
 
 from __future__ import annotations
 
-import os
 import re
 from contextlib import suppress
 from typing import Any
@@ -12,6 +11,7 @@
 from kubernetes import client
 
 from enterprise_gateway.services.processproxies.k8s_client import kubernetes_client
+
 from ..kernels.remotemanager import RemoteKernelManager
 from .k8s import KubernetesProcessProxy
 
diff --git a/enterprise_gateway/services/processproxies/k8s.py b/enterprise_gateway/services/processproxies/k8s.py
index 6dfeddfc5..9892a1534 100644
--- a/enterprise_gateway/services/processproxies/k8s.py
+++ b/enterprise_gateway/services/processproxies/k8s.py
@@ -15,6 +15,7 @@
 from kubernetes.utils.create_from_yaml import create_from_yaml_single_item
 
 from enterprise_gateway.services.processproxies.k8s_client import kubernetes_client
+
 from ..kernels.remotemanager import RemoteKernelManager
 from ..sessions.kernelsessionmanager import KernelSessionManager
 from ..utils.envutils import is_env_true
diff --git a/enterprise_gateway/services/processproxies/k8s_client_factory.py b/enterprise_gateway/services/processproxies/k8s_client_factory.py
index 0b29aa7ce..c0576cc62 100644
--- a/enterprise_gateway/services/processproxies/k8s_client_factory.py
+++ b/enterprise_gateway/services/processproxies/k8s_client_factory.py
@@ -30,7 +30,9 @@ def get_kubernetes_client(self) -> client.ApiClient:
                 )
             else:
                 if is_env_true('EG_USE_REMOTE_CLUSTER'):
-                    self.log.warning(f"Cannot use EG_USE_REMOTE_CLUSTER and EG_SHARED_NAMESPACE at the same time. Using local cluster....")
+                    self.log.warning(
+                        "Cannot use EG_USE_REMOTE_CLUSTER and EG_SHARED_NAMESPACE at the same time. Using local cluster...."
+                    )
 
                 config.load_incluster_config(client_configuration=kubernetes_config)
         else: