diff --git a/README.md b/README.md index 894964c6..dd65773c 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Peel offers the following features for your experiments. | Flink | 1.1.1 | `flink-1.1.1` | | Flink | 1.1.2 | `flink-1.1.2` | | Flink | 1.1.3 | `flink-1.1.3` | +| Flink | 1.1.4 | `flink-1.1.4` | | MapReduce | 1.2.1 | `mapred-1.2.1` | | MapReduce | 2.4.1 | `mapred-2.4.1` | | Spark | 1.3.1 | `spark-1.3.1` | @@ -56,6 +57,8 @@ Peel offers the following features for your experiments. | Spark | 1.6.2 | `spark-1.6.2` | | Spark | 2.0.0 | `spark-2.0.0` | | Spark | 2.0.1 | `spark-2.0.1` | +| Spark | 2.0.2 | `spark-2.0.2` | +| Spark | 2.1.0 | `spark-2.1.0` | | Zookeeper | 3.4.5 | `zookeeper-3.4.5` | | Dstat | 0.7.2 | `dstat-0.7.2` | diff --git a/peel-extensions/src/main/resources/reference.flink-1.1.4.conf b/peel-extensions/src/main/resources/reference.flink-1.1.4.conf new file mode 100644 index 00000000..f685cb68 --- /dev/null +++ b/peel-extensions/src/main/resources/reference.flink-1.1.4.conf @@ -0,0 +1,19 @@ +# include common flink configuration +include "reference.flink.conf" + +system { + flink { + path { + archive.url = "http://archive.apache.org/dist/flink/flink-1.1.4/flink-1.1.4-bin-hadoop2-scala_2.10.tgz" + archive.md5 = "e270a65b0bbaefe5214ac7154b938dc0" + archive.src = ${app.path.downloads}"/flink-1.1.4-bin-hadoop2-scala_2.10.tgz" + home = ${system.flink.path.archive.dst}"/flink-1.1.4" + } + config { + # flink.yaml entries + yaml { + env.pid.dir = "/tmp/flink-1.1.4-pid" + } + } + } +} diff --git a/peel-extensions/src/main/resources/reference.spark-2.0.2.conf b/peel-extensions/src/main/resources/reference.spark-2.0.2.conf new file mode 100644 index 00000000..fe894258 --- /dev/null +++ b/peel-extensions/src/main/resources/reference.spark-2.0.2.conf @@ -0,0 +1,20 @@ +# include common spark configuration +include "reference.spark.conf" + +system { + spark { + path { + archive.url = "http://archive.apache.org/dist/spark/spark-2.0.2/spark-2.0.2-bin-hadoop2.4.tgz" + archive.md5 = "394853fd3e19239b3fc11e40abba6744" + archive.src = ${app.path.downloads}"/spark-2.0.2-bin-hadoop2.4.tgz" + home = ${system.spark.path.archive.dst}"/spark-2.0.2-bin-hadoop2.4" + } + config { + # spark-env.sh entries + env { + # directory where process IDs are stored + SPARK_PID_DIR = "/tmp/spark-2.0.2-pid" + } + } + } +} \ No newline at end of file diff --git a/peel-extensions/src/main/resources/reference.spark-2.1.0.conf b/peel-extensions/src/main/resources/reference.spark-2.1.0.conf new file mode 100644 index 00000000..30a01461 --- /dev/null +++ b/peel-extensions/src/main/resources/reference.spark-2.1.0.conf @@ -0,0 +1,20 @@ +# include common spark configuration +include "reference.spark.conf" + +system { + spark { + path { + archive.url = "http://archive.apache.org/dist/spark/spark-2.1.0/spark-2.1.0-bin-hadoop2.4.tgz" + archive.md5 = "eebfc937e0c3c76174617465d1a8b408" + archive.src = ${app.path.downloads}"/spark-2.1.0-bin-hadoop2.4.tgz" + home = ${system.spark.path.archive.dst}"/spark-2.1.0-bin-hadoop2.4" + } + config { + # spark-env.sh entries + env { + # directory where process IDs are stored + SPARK_PID_DIR = "/tmp/spark-2.1.0-pid" + } + } + } +} \ No newline at end of file diff --git a/peel-extensions/src/main/scala/org/peelframework/extensions.scala b/peel-extensions/src/main/scala/org/peelframework/extensions.scala index 8ee076f9..b86575c3 100644 --- a/peel-extensions/src/main/scala/org/peelframework/extensions.scala +++ b/peel-extensions/src/main/scala/org/peelframework/extensions.scala @@ -207,6 +207,14 @@ class extensions extends ApplicationContextAware { mc = ctx.getBean(classOf[Mustache.Compiler]) ) + @Bean(name = Array("flink-1.1.4")) + def `flink-1.1.4`: Flink = new Flink( + version = "1.1.4", + configKey = "flink", + lifespan = Lifespan.EXPERIMENT, + mc = ctx.getBean(classOf[Mustache.Compiler]) + ) + // Spark @Bean(name = Array("spark-1.3.1")) @@ -273,6 +281,22 @@ class extensions extends ApplicationContextAware { mc = ctx.getBean(classOf[Mustache.Compiler]) ) + @Bean(name = Array("spark-2.0.2")) + def `spark-2.0.2`: Spark = new Spark( + version = "2.0.2", + configKey = "spark", + lifespan = Lifespan.EXPERIMENT, + mc = ctx.getBean(classOf[Mustache.Compiler]) + ) + + @Bean(name = Array("spark-2.1.0")) + def `spark-2.1.0`: Spark = new Spark( + version = "2.1.0", + configKey = "spark", + lifespan = Lifespan.EXPERIMENT, + mc = ctx.getBean(classOf[Mustache.Compiler]) + ) + // DStat @Bean(name = Array("dstat-0.7.2"))