diff --git a/README.md b/README.md index 5e17d5d7..cf8e4e89 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ Peel offers the following features for your experiments. | Flink | 1.2.1 | `flink-1.2.1` | | Flink | 1.3.0 | `flink-1.3.0` | | Flink | 1.3.1 | `flink-1.3.1` | +| Flink | 1.3.2 | `flink-1.3.2` | +| Flink | 1.4.0 | `flink-1.4.0` | | MapReduce | 1.2.1 | `mapred-1.2.1` | | MapReduce | 2.4.1 | `mapred-2.4.1` | | Spark | 1.3.1 | `spark-1.3.1` | @@ -69,6 +71,7 @@ Peel offers the following features for your experiments. | Spark | 2.1.0 | `spark-2.1.0` | | Spark | 2.1.1 | `spark-2.1.1` | | Spark | 2.2.0 | `spark-2.2.0` | +| Spark | 2.2.1 | `spark-2.2.1` | | Zookeeper | 3.4.5 | `zookeeper-3.4.5` | | Dstat | 0.7.2 | `dstat-0.7.2` | | Dstat | 0.7.3 | `dstat-0.7.3` | diff --git a/docs/_manual/experiments-definitions.md b/docs/_manual/experiments-definitions.md index f738d11c..5cf566b8 100644 --- a/docs/_manual/experiments-definitions.md +++ b/docs/_manual/experiments-definitions.md @@ -123,6 +123,11 @@ The `peel-extensions` module ships with several *System* implementations. The fo | Flink | 1.1.3 | flink-1.1.3 | | Flink | 1.1.4 | flink-1.1.4 | | Flink | 1.2.0 | flink-1.2.0 | +| Flink | 1.2.1 | flink-1.2.1 | +| Flink | 1.3.0 | flink-1.3.0 | +| Flink | 1.3.1 | flink-1.3.1 | +| Flink | 1.3.2 | flink-1.3.2 | +| Flink | 1.4.0 | flink-1.4.0 | | MapReduce | 1.2.1 | mapred-1.2.1 | | MapReduce | 2.4.1 | mapred-2.4.1 | | Spark | 1.3.1 | spark-1.3.1 | @@ -136,6 +141,9 @@ The `peel-extensions` module ships with several *System* implementations. The fo | Spark | 2.0.1 | spark-2.0.1 | | Spark | 2.0.2 | spark-2.0.2 | | Spark | 2.1.0 | spark-2.1.0 | +| Spark | 2.1.1 | spark-2.1.1 | +| Spark | 2.2.0 | spark-2.2.0 | +| Spark | 2.2.1 | spark-2.2.1 | | Zookeeper | 3.4.5 | zookeeper-3.4.5 | | Dstat | 0.7.2 | dstat-0.7.2 | | Dstat | 0.7.3 | dstat-0.7.3 | diff --git a/peel-extensions/src/main/resources/reference.flink-1.3.2.conf b/peel-extensions/src/main/resources/reference.flink-1.3.2.conf new file mode 100644 index 00000000..8e0068e5 --- /dev/null +++ b/peel-extensions/src/main/resources/reference.flink-1.3.2.conf @@ -0,0 +1,19 @@ +# include common flink configuration +include "reference.flink.conf" + +system { + flink { + path { + archive.url = "http://archive.apache.org/dist/flink/flink-1.3.2/flink-1.3.2-bin-hadoop2-scala_2.10.tgz" + archive.md5 = "f397881ec7629027281fdaa1cb7f8085" + archive.src = ${app.path.downloads}"/flink-1.3.2-bin-hadoop2-scala_2.10.tgz" + home = ${system.flink.path.archive.dst}"/flink-1.3.2" + } + config { + # flink.yaml entries + yaml { + env.pid.dir = "/tmp/flink-1.3.2-pid" + } + } + } +} diff --git a/peel-extensions/src/main/resources/reference.flink-1.4.0.conf b/peel-extensions/src/main/resources/reference.flink-1.4.0.conf new file mode 100644 index 00000000..2ef3ac94 --- /dev/null +++ b/peel-extensions/src/main/resources/reference.flink-1.4.0.conf @@ -0,0 +1,19 @@ +# include common flink configuration +include "reference.flink.conf" + +system { + flink { + path { + archive.url = "http://archive.apache.org/dist/flink/flink-1.4.0/flink-1.4.0-bin-hadoop27-scala_2.11.tgz" + archive.md5 = "4ad7d053f76337e02823f2ca6479a2e7" + archive.src = ${app.path.downloads}"/flink-1.4.0-bin-hadoop27-scala_2.11.tgz" + home = ${system.flink.path.archive.dst}"/flink-1.4.0" + } + config { + # flink.yaml entries + yaml { + env.pid.dir = "/tmp/flink-1.4.0-pid" + } + } + } +} diff --git a/peel-extensions/src/main/resources/reference.spark-2.2.1.conf b/peel-extensions/src/main/resources/reference.spark-2.2.1.conf new file mode 100644 index 00000000..a57b1165 --- /dev/null +++ b/peel-extensions/src/main/resources/reference.spark-2.2.1.conf @@ -0,0 +1,20 @@ +# include common spark configuration +include "reference.spark.conf" + +system { + spark { + path { + archive.url = "http://archive.apache.org/dist/spark/spark-2.2.1/spark-2.2.1-bin-hadoop2.7.tgz" + archive.md5 = "c0081f6076070f0a6c6a607c71ac7e95" + archive.src = ${app.path.downloads}"/spark-2.2.1-bin-hadoop2.7.tgz" + home = ${system.spark.path.archive.dst}"/spark-2.2.1-bin-hadoop2.7" + } + config { + # spark-env.sh entries + env { + # directory where process IDs are stored + SPARK_PID_DIR = "/tmp/spark-2.2.1-pid" + } + } + } +} \ No newline at end of file diff --git a/peel-extensions/src/main/scala/org/peelframework/extensions.scala b/peel-extensions/src/main/scala/org/peelframework/extensions.scala index 71978293..a5583be2 100644 --- a/peel-extensions/src/main/scala/org/peelframework/extensions.scala +++ b/peel-extensions/src/main/scala/org/peelframework/extensions.scala @@ -271,6 +271,22 @@ class extensions extends ApplicationContextAware { mc = ctx.getBean(classOf[Mustache.Compiler]) ) + @Bean(name = Array("flink-1.3.2")) + def `flink-1.3.2`: Flink = new Flink( + version = "1.3.2", + configKey = "flink", + lifespan = Lifespan.EXPERIMENT, + mc = ctx.getBean(classOf[Mustache.Compiler]) + ) + + @Bean(name = Array("flink-1.4.0")) + def `flink-1.4.0`: Flink = new Flink( + version = "1.4.0", + configKey = "flink", + lifespan = Lifespan.EXPERIMENT, + mc = ctx.getBean(classOf[Mustache.Compiler]) + ) + // Spark @Bean(name = Array("spark-1.3.1")) @@ -369,6 +385,14 @@ class extensions extends ApplicationContextAware { mc = ctx.getBean(classOf[Mustache.Compiler]) ) + @Bean(name = Array("spark-2.2.1")) + def `spark-2.2.1`: Spark = new Spark( + version = "2.2.1", + configKey = "spark", + lifespan = Lifespan.EXPERIMENT, + mc = ctx.getBean(classOf[Mustache.Compiler]) + ) + // DStat @Bean(name = Array("dstat-0.7.2"))