Adds Flink 1.3.2, 1.4.0 and Spark 2.2.1.

peelframework · Jan 31, 2018 · 3e0434a · 3e0434a
1 parent ec280e0
commit 3e0434a
Show file tree

Hide file tree

Showing 6 changed files with 93 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -54,6 +54,8 @@ Peel offers the following features for your experiments.
 | Flink            | 1.2.1          | `flink-1.2.1`     |
 | Flink            | 1.3.0          | `flink-1.3.0`     |
 | Flink            | 1.3.1          | `flink-1.3.1`     |
+| Flink            | 1.3.2          | `flink-1.3.2`     |
+| Flink            | 1.4.0          | `flink-1.4.0`     |
 | MapReduce        | 1.2.1          | `mapred-1.2.1`    |
 | MapReduce        | 2.4.1          | `mapred-2.4.1`    |
 | Spark            | 1.3.1          | `spark-1.3.1`     |
@@ -69,6 +71,7 @@ Peel offers the following features for your experiments.
 | Spark            | 2.1.0          | `spark-2.1.0`     |
 | Spark            | 2.1.1          | `spark-2.1.1`     |
 | Spark            | 2.2.0          | `spark-2.2.0`     |
+| Spark            | 2.2.1          | `spark-2.2.1`     |
 | Zookeeper        | 3.4.5          | `zookeeper-3.4.5` |
 | Dstat            | 0.7.2          | `dstat-0.7.2`     |
 | Dstat            | 0.7.3          | `dstat-0.7.3`     |

diff --git a/docs/_manual/experiments-definitions.md b/docs/_manual/experiments-definitions.md
@@ -123,6 +123,11 @@ The `peel-extensions` module ships with several *System* implementations. The fo
 | Flink            | 1.1.3          | flink-1.1.3     |
 | Flink            | 1.1.4          | flink-1.1.4     |
 | Flink            | 1.2.0          | flink-1.2.0     |
+| Flink            | 1.2.1          | flink-1.2.1     |
+| Flink            | 1.3.0          | flink-1.3.0     |
+| Flink            | 1.3.1          | flink-1.3.1     |
+| Flink            | 1.3.2          | flink-1.3.2     |
+| Flink            | 1.4.0          | flink-1.4.0     |
 | MapReduce        | 1.2.1          | mapred-1.2.1    |
 | MapReduce        | 2.4.1          | mapred-2.4.1    |
 | Spark            | 1.3.1          | spark-1.3.1     |
@@ -136,6 +141,9 @@ The `peel-extensions` module ships with several *System* implementations. The fo
 | Spark            | 2.0.1          | spark-2.0.1     |
 | Spark            | 2.0.2          | spark-2.0.2     |
 | Spark            | 2.1.0          | spark-2.1.0     |
+| Spark            | 2.1.1          | spark-2.1.1     |
+| Spark            | 2.2.0          | spark-2.2.0     |
+| Spark            | 2.2.1          | spark-2.2.1     |
 | Zookeeper        | 3.4.5          | zookeeper-3.4.5 |
 | Dstat            | 0.7.2          | dstat-0.7.2     |
 | Dstat            | 0.7.3          | dstat-0.7.3     |

diff --git a/peel-extensions/src/main/resources/reference.flink-1.3.2.conf b/peel-extensions/src/main/resources/reference.flink-1.3.2.conf
@@ -0,0 +1,19 @@
+# include common flink configuration
+include "reference.flink.conf"
+
+system {
+  flink {
+    path {
+      archive.url = "http://archive.apache.org/dist/flink/flink-1.3.2/flink-1.3.2-bin-hadoop2-scala_2.10.tgz"
+      archive.md5 = "f397881ec7629027281fdaa1cb7f8085"
+      archive.src = ${app.path.downloads}"/flink-1.3.2-bin-hadoop2-scala_2.10.tgz"
+      home = ${system.flink.path.archive.dst}"/flink-1.3.2"
+    }
+    config {
+      # flink.yaml entries
+      yaml {
+        env.pid.dir = "/tmp/flink-1.3.2-pid"
+      }
+    }
+  }
+}
diff --git a/peel-extensions/src/main/resources/reference.flink-1.4.0.conf b/peel-extensions/src/main/resources/reference.flink-1.4.0.conf
@@ -0,0 +1,19 @@
+# include common flink configuration
+include "reference.flink.conf"
+
+system {
+  flink {
+    path {
+      archive.url = "http://archive.apache.org/dist/flink/flink-1.4.0/flink-1.4.0-bin-hadoop27-scala_2.11.tgz"
+      archive.md5 = "4ad7d053f76337e02823f2ca6479a2e7"
+      archive.src = ${app.path.downloads}"/flink-1.4.0-bin-hadoop27-scala_2.11.tgz"
+      home = ${system.flink.path.archive.dst}"/flink-1.4.0"
+    }
+    config {
+      # flink.yaml entries
+      yaml {
+        env.pid.dir = "/tmp/flink-1.4.0-pid"
+      }
+    }
+  }
+}
diff --git a/peel-extensions/src/main/resources/reference.spark-2.2.1.conf b/peel-extensions/src/main/resources/reference.spark-2.2.1.conf
@@ -0,0 +1,20 @@
+# include common spark configuration
+include "reference.spark.conf"
+
+system {
+    spark {
+        path {
+            archive.url = "http://archive.apache.org/dist/spark/spark-2.2.1/spark-2.2.1-bin-hadoop2.7.tgz"
+            archive.md5 = "c0081f6076070f0a6c6a607c71ac7e95"
+            archive.src = ${app.path.downloads}"/spark-2.2.1-bin-hadoop2.7.tgz"
+            home = ${system.spark.path.archive.dst}"/spark-2.2.1-bin-hadoop2.7"
+        }
+        config {
+            # spark-env.sh entries
+            env {
+                # directory where process IDs are stored
+                SPARK_PID_DIR = "/tmp/spark-2.2.1-pid"
+            }
+        }
+    }
+}
diff --git a/peel-extensions/src/main/scala/org/peelframework/extensions.scala b/peel-extensions/src/main/scala/org/peelframework/extensions.scala
@@ -271,6 +271,22 @@ class extensions extends ApplicationContextAware {
     mc           = ctx.getBean(classOf[Mustache.Compiler])
   )
 
+  @Bean(name = Array("flink-1.3.2"))
+  def `flink-1.3.2`: Flink = new Flink(
+    version      = "1.3.2",
+    configKey    = "flink",
+    lifespan     = Lifespan.EXPERIMENT,
+    mc           = ctx.getBean(classOf[Mustache.Compiler])
+  )
+
+  @Bean(name = Array("flink-1.4.0"))
+  def `flink-1.4.0`: Flink = new Flink(
+    version      = "1.4.0",
+    configKey    = "flink",
+    lifespan     = Lifespan.EXPERIMENT,
+    mc           = ctx.getBean(classOf[Mustache.Compiler])
+  )
+
   // Spark
 
   @Bean(name = Array("spark-1.3.1"))
@@ -369,6 +385,14 @@ class extensions extends ApplicationContextAware {
     mc           = ctx.getBean(classOf[Mustache.Compiler])
   )
 
+  @Bean(name = Array("spark-2.2.1"))
+  def `spark-2.2.1`: Spark = new Spark(
+    version      = "2.2.1",
+    configKey    = "spark",
+    lifespan     = Lifespan.EXPERIMENT,
+    mc           = ctx.getBean(classOf[Mustache.Compiler])
+  )
+
   // DStat
 
   @Bean(name = Array("dstat-0.7.2"))