Merge branch 'dromara:dev' into dev

.all-contributorsrc

@@ -133,6 +133,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "stdnt-xiao",
+      "name": "stdnt-xiao",
+      "avatar_url": "https://avatars.githubusercontent.com/u/20313983?v=4",
+      "profile": "https://github.com/stdnt-xiao",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

README.md

@@ -100,6 +100,9 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/rugalcrimson"><img src="https://avatars.githubusercontent.com/u/15381951?v=4?s=100" width="100px;" alt="rugalcrimson"/><br /><sub><b>rugalcrimson</b></sub></a><br /><a href="https://github.com/dromara/CloudEon/commits?author=rugalcrimson" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/q443048756"><img src="https://avatars.githubusercontent.com/u/47968271?v=4?s=100" width="100px;" alt="buer"/><br /><sub><b>buer</b></sub></a><br /><a href="https://github.com/dromara/CloudEon/commits?author=q443048756" title="Code">💻</a></td>
     </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/stdnt-xiao"><img src="https://avatars.githubusercontent.com/u/20313983?v=4?s=100" width="100px;" alt="stdnt-xiao"/><br /><sub><b>stdnt-xiao</b></sub></a><br /><a href="https://github.com/dromara/CloudEon/commits?author=stdnt-xiao" title="Code">💻</a></td>
+    </tr>
   </tbody>
 </table>
 

cloudeon-assembly/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>cloudeon</artifactId>
         <groupId>org.dromara</groupId>
-        <version>v1.2.0</version>
+        <version>v1.3.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

cloudeon-docs/docs/logserver.md

@@ -0,0 +1,8 @@
+# 日志收集
+
+## 前提
+需要安装es服务和filebeat服务，filebeat角色实例应该每个节点都安装。
+
+## 使用
+安装成功后，可以在页面中查看对应服务角色实例的日志
+![log.png](images%2Flog.png)

cloudeon-docs/docs/安装部署/minikube.md

@@ -0,0 +1,54 @@
+# minikube部署轻量kubernetes
+## 1. 简介
+
+本文介绍使用minikube部署轻量kubernetes，用于CloudEon测试环境搭建。
+
+## 2. 环境准备
+安装minikube、vmware。
+```text
+注：不能使用docker作为虚拟节点驱动，因为无法直接访问节点ip和端口。
+```
+
+## 3. 部署kubernetes环境
+```shell
+minikube start --vm-driver=vmware --nodes 3
+```
+
+## 4. 设置所有节点root远程登录密码
+### 4.1 查看kubernetes节点列表
+```shell
+minikube node list
+```
+```text
+minikube	192.168.142.135
+minikube-m02	192.168.142.136
+minikube-m03	192.168.142.137
+```
+### 4.2 登录kubernetes子节点
+```shell
+minikube ssh -n minikube-m03
+```
+### 4.3 设置root密码
+```shell
+sudo passwd
+```
+### 4.4 允许root用户远程登录
+```shell
+sudo vi /etc/ssh/sshd_config
+```
+修改
+```shell
+PermitRootLogin yes
+```
+重启sshd服务
+```shell
+sudo systemctl restart sshd
+```
+### 4.5 验证root用户远程登录
+```shell
+ssh root@minikube-m03
+```
+## 5. 获取kubernetes的kubeConfig
+```shell
+cat ~/.kube/config
+```

cloudeon-docs/docs/支持组件/HueOnK8s.md

@@ -0,0 +1,15 @@
+# Hue on  K8s
+安装服务
+![img_2.png](../images/hue/img_2.png)
+分配角色实例到指定节点安装。
+![img_3.png](../images/hue/img_3.png)
+
+修改初始化配置，需要填写mysql信息，数据库得提前自行准备好。安装过程会自动初始化hue的表结构。
+![img_4.png](../images/hue/img_4.png)
+安装成功后，打开web地址，默认密码是admin/admin
+![img_5.png](../images/hue/img_5.png)
+查看hdfs文件
+![img.png](../images/hue/img.png)
+查看hive表数据
+![img_1.png](../images/hue/img_1.png)
+

cloudeon-docs/mkdocs.yml

@@ -61,6 +61,7 @@ nav:
       - 安装部署/install_pre.md
       - 安装部署/docker.md
       - 安装部署/local.md
+      - 安装部署/minikube.md
   - quickstart.md
   - 支持组件:
       - 支持组件/supportservice.md
@@ -83,12 +84,14 @@ nav:
       - 支持组件/IcebergOnK8s.md
       - 支持组件/TrinoOnK8s.md
       - 支持组件/SeatunnelOnK8s.md
+      - 支持组件/HueOnK8s.md
   - component_extension.md
   - 社区贡献:
       - 社区贡献/communicate.md
       - 社区贡献/contribute.md
       - 社区贡献/develop.md
       - 社区贡献/windows-develop.md
   - sponsor.md
+  - logserver.md
   - FAQ.md
   - Roadmap.md

cloudeon-server/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.dromara</groupId>
         <artifactId>cloudeon</artifactId>
-        <version>v1.2.0</version>
+        <version>v1.3.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>cloudeon-server</artifactId>

cloudeon-server/src/main/java/org/dromara/cloudeon/processor/ConfigTask.java

@@ -230,7 +230,9 @@ private void copyDependenceServiceConf(String stackServiceName, String[] depServ
                 ||stackServiceName.equalsIgnoreCase(Constant.TRINO_SERVICE_NAME)
                 ||stackServiceName.equalsIgnoreCase(Constant.AMORO_SERVICE_NAME)
                 ||stackServiceName.equalsIgnoreCase(Constant.HIVE_SERVICE_NAME)
-                ||stackServiceName.equalsIgnoreCase(Constant.DINKY_SERVICE_NAME)) {
+                ||stackServiceName.equalsIgnoreCase(Constant.DINKY_SERVICE_NAME)
+            ||stackServiceName.equalsIgnoreCase(Constant.HUE_SERVICE_NAME)
+        ) {
             List<Integer> instanceIds = Arrays.stream(depServiceInstanceIds).map(new Function<String, Integer>() {
                 @Override
                 public Integer apply(String s) {

cloudeon-server/src/main/java/org/dromara/cloudeon/utils/Constant.java

@@ -42,6 +42,7 @@ public class Constant {
     public static final String DOLPHINSCHEDULER_SERVICE_NAME = "DOLPHINSCHEDULER";
     public static final String FLINK_SERVICE_NAME = "FLINK";
     public static final String DINKY_SERVICE_NAME = "DINKY";
+    public static final String HUE_SERVICE_NAME = "HUE";
     public static final String DORIS_SERVICE_NAME = "DORIS";
     public static final String SPARK_SERVICE_NAME = "SPARK";
     public static final String HDFS_SERVICE_NAME = "HDFS";

cloudeon-server/src/test/java/org/dromara/cloudeon/fabric8/ExecuteCommandOnPodExample.java

@@ -24,6 +24,8 @@
 import io.fabric8.kubernetes.client.KubernetesClientBuilder;
 import io.fabric8.kubernetes.client.dsl.ExecListener;
 import io.fabric8.kubernetes.client.dsl.ExecWatch;
+import lombok.AllArgsConstructor;
+import lombok.Data;
 import lombok.SneakyThrows;
 
 import java.io.ByteArrayOutputStream;
@@ -100,15 +102,62 @@ public void onClose(int code, String reason) {
 
   public static void main(String[] args) {
 
-    final String pod = "hadoop-hdfs-datanode-hdfs1-9f5f7c69f-6x2wg";
+    final String pod = "nginx";
     final String namespace = "default";
-    final String command = "ls -lQ --color=never --full-time /opt/edp/hdfs1/conf";
+    String basePath = "/etc";
+    final String command = "ls -lQ --color=never --full-time " + basePath;
 //    final String command = "cat /opt/edp/hdfs1/conf/core-site.xml";
 
     try (ExecuteCommandOnPodExample example = new ExecuteCommandOnPodExample()) {
       String cmdOutput = example.execCommandOnPod(pod, namespace, command.split(" "));
       System.out.println(cmdOutput);
+
+      // 解析命令执行结果并封装到 JavaBean 中
+      String[] lines = cmdOutput.split("\\r?\\n");
+      for (String line : lines) {
+        if (!line.isEmpty() && !line.startsWith("total")) {
+          FileEntry fileEntry = parseLineToBean(line, basePath);
+          // 处理每个文件条目
+          System.out.println(fileEntry);
+        }
+      }
+    }
+
+  }
+
+  private static FileEntry parseLineToBean(String line, String basePath) {
+    String[] parts = line.split("\\s+");
+    String permissions = parts[0];
+    String owner = parts[2];
+    String group = parts[3];
+    String size = parts[4];
+    String date = parts[5] + " " + parts[6] + " " + parts[7];
+    String name = parts[8].replace("\"", "");
+    // 判断是否目录
+    boolean isDir = false;
+    if (line.startsWith("d")) {
+      isDir = true;
     }
 
+    String fullPath = basePath + "/" + name;
+
+    // 创建 FileEntry 对象并设置属性
+    FileEntry fileEntry = new FileEntry(isDir, fullPath, permissions, owner, group, size, date, name);
+    return fileEntry;
+  }
+
+  @Data
+  @AllArgsConstructor
+  static class FileEntry {
+
+    boolean isDir;
+    String fullPath;
+    String permissions;
+    String owner;
+    String group;
+    String size;
+    String date;
+    String name;
+
   }
 }

cloudeon-stack/EDP-1.0.0/dinky/render/bootstrap-dinky.sh.ftl

@@ -33,4 +33,4 @@ nohup java $JAVA_OPTS \
 -Dlogging.config=$DINKY_CONF/log4j2.xml  -Dspring.config.location=$DINKY_CONF/dinky-application.yaml,$DINKY_CONF/application.properties com.dlink.Dlink     > $log 2>&1 &
 
 echo $! > $pid
-tail -f /dev/null
+tail -f /opt/edp/${service.serviceName}/log/dinky-$HOSTNAME.out

cloudeon-stack/EDP-1.0.0/hdfs/render/hadoop-hdfs-env.sh.ftl

@@ -105,5 +105,5 @@ export JOURNALNODE_RPC_PORT=${conf['journalnode.rpc-port']}
 export DATANODE_DATA_DIRS=/opt/edp/${service.serviceName}/data/datanode
 
 # Export dfs.namenode.name.dir
-export NAMENODE_DATA_DIRS=/opt/edp/${service.serviceName}/data/namenode
+export DATANODE_DATA_DIRS=${conf['dfs.datanode.data.dir']}
 

cloudeon-stack/EDP-1.0.0/hdfs/render/hdfs-site.xml.ftl

@@ -48,7 +48,7 @@
 </#if>
 
 <#--handle data dir-->
-<@property "dfs.datanode.data.dir" "/opt/edp/${serviceName}/data/datanode"/>
+<@property "dfs.datanode.data.dir" "${conf['dfs.datanode.data.dir']}"/>
 <@property "dfs.namenode.name.dir" "/opt/edp/${serviceName}/data/namenode"/>
 <@property "dfs.journalnode.edits.dir" "/opt/edp/${serviceName}/data/journal"/>
 

cloudeon-stack/EDP-1.0.0/hdfs/service-info.yaml

@@ -192,6 +192,15 @@ configurations:
     tag: "常用参数"
 
 
+  - name: dfs.datanode.data.dir
+    recommendExpression: /opt/edp/hdfs/data/datanode1
+    valueType: InputString
+    description: HDFS数据目录，多个目录使用","分割。子目录需datanode所在宿主机的/opt/edp/hdfs/data目录下。默认会在datanode初始化时自动创建，如果希望挂载数据盘，需要自行提前创建并设置目录权限为和容器内操作用户相同的uid。如容器内uid为1002，则需在宿主机上执行 chown -R  1002:1002 data/datanode1
+    confFile:  "hdfs-site.xml"
+    configurableInWizard: true
+    tag: "常用参数"
+
+
   - name: dfs.datanode.data.dir.perm
     recommendExpression: 755
     valueType: InputNumber

cloudeon-stack/EDP-1.0.0/hive/k8s/hive-metastore.yaml.ftl

@@ -45,6 +45,8 @@ spec:
         env:
           - name: "SERVICE_NAME"
             value: "metastore"
+          - name: HADOOP_CONF_DIR
+            value: /opt/edp/${service.serviceName}/conf
           - name: "HIVE_CONF_DIR"
             value: "/opt/edp/${service.serviceName}/conf"
           - name: MEM_LIMIT

cloudeon-stack/EDP-1.0.0/hive/k8s/hive-server2.yaml.ftl

@@ -45,6 +45,8 @@ spec:
         env:
           - name: "SERVICE_NAME"
             value: "server2"
+          - name: HADOOP_CONF_DIR
+            value: /opt/edp/${service.serviceName}/conf
           - name: "HIVE_CONF_DIR"
             value: "/opt/edp/${service.serviceName}/conf"
           - name: MEM_LIMIT

cloudeon-stack/EDP-1.0.0/hue/alert-rule.yaml

@@ -0,0 +1,7 @@
+rules:
+    - alert: AmoroAms进程存活
+      promql: up{job="ams"} != 1
+      alertLevel: exception
+      serviceRoleName: AMORO_AMS
+      alertAdvice: AmoroAms宕机，请重新启动
+      alertInfo: "{{ $labels.job }}的{{ $labels.instance }}实例产生告警"

cloudeon-stack/EDP-1.0.0/hue/docker/Dockerfile

@@ -0,0 +1,2 @@
+FROM gethue/hue:4.11.0
+

cloudeon-stack/EDP-1.0.0/hue/docker/build.sh

@@ -0,0 +1,4 @@
+docker build -f Dockerfile -t hue:4.11.0 .
+docker tag   hue:4.11.0  registry.cn-hangzhou.aliyuncs.com/udh/hue:4.11.0
+docker push registry.cn-hangzhou.aliyuncs.com/udh/hue:4.11.0
+