Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[add] add paimonwriter.json to docs #1194

Merged
merged 10 commits into from
Jan 24, 2025
Merged

[add] add paimonwriter.json to docs #1194

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0ef26ed
[feature][plugin][paimonwriter] add paimon format file writer
awol2005ex Jan 21, 2025
5a6c92e
fixed timestamp,decimal,null type value bug
awol2005ex Jan 21, 2025
992de9d
1.change author
awol2005ex Jan 22, 2025
9c7e7ab
Merge branch 'master' into awol2005ex-br-paimonwriter-2025-01-21
awol2005ex Jan 23, 2025
1784cf0
[add] add paimonwriter.json to docs
awol2005ex Jan 23, 2025
30ce1e7
[add] add the way of create paimon table before test to paimonwriter.md
awol2005ex Jan 23, 2025
637f861
[add] add the way of create paimon table before test to paimonwriter.md
awol2005ex Jan 23, 2025
a59ab8f
[add] add the way of create paimon table before test to paimonwriter.…
awol2005ex Jan 24, 2025
387f15e
fixed: the bug that no hadoop conf when use s3a fielsystem
awol2005ex Jan 24, 2025
1b4dbec
add minio example to paimonwriter.md
awol2005ex Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions docs/assets/jobs/paimonwriter.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
{
"job": {
"setting": {
"speed": {
"channel": 3
},
"errorLimit": {
"record": 0,
"percentage": 0
}
},
"content": [
{
"reader": {
"name": "rdbmsreader",
"parameter": {
"username": "root",
"password": "root",
"column": [
"*"
],
"connection": [
{
"querySql": [
"select 1+0 id ,'test1' as name"
],
"jdbcUrl": ["jdbc:mysql://localhost:3306/ruoyi_vue_camunda?allowPublicKeyRetrieval=true",]
}
],
"fetchSize": 1024
}
},
"writer": {
"name": "paimonwriter",
"parameter": {
"dbName": "test",
"tableName": "test2",
"writeMode": "truncate",
"paimonConfig": {
"warehouse": "file:///g:/paimon",
"metastore": "filesystem"
}
}
}
}
]
}
}
266 changes: 266 additions & 0 deletions docs/writer/paimonwriter.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,272 @@ Paimon Writer 提供向 已有的paimon表写入数据的能力。
### paimonConfig

`paimonConfig` 里可以配置与 Paimon catalog和Hadoop 相关的一些高级参数，比如HA的配置

本地目录创建paimon表

pom.xml

```xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>com.test</groupId>
<artifactId>paimon-java-api-test</artifactId>
<version>1.0-SNAPSHOT</version>

<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.version>3.2.4</hadoop.version>
<woodstox.version>7.0.0</woodstox.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-bundle</artifactId>
<version>1.0.0</version>
</dependency>


<dependency>
awol2005ex marked this conversation as resolved.
Show resolved Hide resolved
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.woodstox</groupId>
<artifactId>woodstox-core</artifactId>
</exclusion>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
<exclusion>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.woodstox</groupId>
<artifactId>woodstox-core</artifactId>
</exclusion>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
<exclusion>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
</exclusions>
</dependency>


<dependency>
<groupId>com.fasterxml.woodstox</groupId>
<artifactId>woodstox-core</artifactId>
<version>${woodstox.version}</version>
</dependency>
</dependencies>
</project>
```

```java

import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.CatalogFactory;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.fs.Path;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.types.DataTypes;

import java.util.HashMap;
import java.util.Map;

public class CreatePaimonTable {

public static Catalog createFilesystemCatalog() {
CatalogContext context = CatalogContext.create(new Path("file:///g:/paimon"));
return CatalogFactory.createCatalog(context);
}

public static void main(String[] args) {
Schema.Builder schemaBuilder = Schema.newBuilder();
schemaBuilder.primaryKey("id");
schemaBuilder.column("id", DataTypes.INT());
schemaBuilder.column("name", DataTypes.STRING());
Map<String, String> options = new HashMap<>();
options.put("bucket", "1");//由于paimon java api 限制需要bucket>0
options.put("bucket-key", "id");
options.put("file.format", "orc");
options.put("file.compression", "lz4");
options.put("lookup.cache-spill-compression", "lz4");
options.put("spill-compression", "LZ4");
options.put("orc.compress", "lz4");
options.put("manifest.format", "orc");

schemaBuilder.options(options);
Schema schema = schemaBuilder.build();

Identifier identifier = Identifier.create("test", "test2");
try {
Catalog catalog = CreatePaimonTable.createFilesystemCatalog();
awol2005ex marked this conversation as resolved.
Show resolved Hide resolved
catalog.createTable(identifier, schema, true);
} catch (Catalog.TableAlreadyExistException e) {
e.printStackTrace();
} catch (Catalog.DatabaseNotExistException e) {
e.printStackTrace();
}


}
}


```

Spark 或者 flink 环境创建表

```sql
CREATE TABLE if not exists test.test2(id int ,name string) tblproperties (
'primary-key' = 'id',
'bucket' = '1',
'bucket-key' = 'id'
'file.format'='orc',
'file.compression'='lz4',
'lookup.cache-spill-compression'='lz4',
'spill-compression'='LZ4',
'orc.compress'='lz4',
'manifest.format'='orc'
)

```


```json
{
"name": "paimonwriter",
Expand Down
Loading