From 6a5e0a899a8d9471893f52b424c05463a5701943 Mon Sep 17 00:00:00 2001 From: Alexander Alexandrov Date: Sun, 16 Aug 2015 12:03:43 +0300 Subject: [PATCH] RunState changes. * Removed 'name' parameter from base trait. * Removed 'suiteName' and 'name' parameters from Flink and Spark State. * Renamed 'RunState' to 'GenericRunState' in ETL package. * Injecting 'suiteName' and 'name' to 'GenericRunState' JSON before parsing. --- .../core/beans/experiment/Experiment.scala | 1 - .../{RunState.scala => GenericRunState.scala} | 23 ++++++++++--------- .../etl/traverser/SuiteTraverser.scala | 17 +++++++++----- .../beans/experiment/FlinkExperiment.scala | 9 +++----- .../beans/experiment/SparkExperiment.scala | 9 +++----- 5 files changed, 29 insertions(+), 30 deletions(-) rename peel-core/src/main/scala/org/peelframework/core/results/etl/{RunState.scala => GenericRunState.scala} (74%) diff --git a/peel-core/src/main/scala/org/peelframework/core/beans/experiment/Experiment.scala b/peel-core/src/main/scala/org/peelframework/core/beans/experiment/Experiment.scala index 716a9575..edc8f74a 100644 --- a/peel-core/src/main/scala/org/peelframework/core/beans/experiment/Experiment.scala +++ b/peel-core/src/main/scala/org/peelframework/core/beans/experiment/Experiment.scala @@ -131,7 +131,6 @@ object Experiment { /** Representation of the state of a run. */ trait RunState { - val name : String val runnerID : String val runnerName : String val runnerVersion : String diff --git a/peel-core/src/main/scala/org/peelframework/core/results/etl/RunState.scala b/peel-core/src/main/scala/org/peelframework/core/results/etl/GenericRunState.scala similarity index 74% rename from peel-core/src/main/scala/org/peelframework/core/results/etl/RunState.scala rename to peel-core/src/main/scala/org/peelframework/core/results/etl/GenericRunState.scala index 2bfddbb6..b33c3e29 100644 --- a/peel-core/src/main/scala/org/peelframework/core/results/etl/RunState.scala +++ b/peel-core/src/main/scala/org/peelframework/core/results/etl/GenericRunState.scala @@ -15,6 +15,7 @@ */ package org.peelframework.core.results.etl +import org.peelframework.core.beans.experiment.Experiment import spray.json.{NullOptions, DefaultJsonProtocol} /** A case class representing the common fields ensured to be appearing in all `state.json` files. @@ -28,19 +29,19 @@ import spray.json.{NullOptions, DefaultJsonProtocol} * @param runExitCode The exit code of this run (0 indicates success). * @param runTime The runtime (in milliseconds) of this run. */ -case class RunState( - name: String, - suiteName: String, - command: String, - runnerID: String, - runnerName: String, - runnerVersion: String, - runExitCode: Option[Int] = None, - runTime: Long = 0) {} +case class GenericRunState( + name : String, + suiteName : String, + command : String, + runnerID : String, + runnerName : String, + runnerVersion : String, + var runExitCode : Option[Int] = None, + var runTime : Long = 0) extends Experiment.RunState -/** An object for [[org.peelframework.core.results.etl.RunState RunState]] <-> JSON format conversion. +/** An object for [[GenericRunState]] <-> JSON format conversion. * */ object RunStateProtocol extends DefaultJsonProtocol with NullOptions { - implicit val stateFormat = jsonFormat8(RunState) + implicit val stateFormat = jsonFormat8(GenericRunState) } diff --git a/peel-core/src/main/scala/org/peelframework/core/results/etl/traverser/SuiteTraverser.scala b/peel-core/src/main/scala/org/peelframework/core/results/etl/traverser/SuiteTraverser.scala index ab22838b..a57c534a 100644 --- a/peel-core/src/main/scala/org/peelframework/core/results/etl/traverser/SuiteTraverser.scala +++ b/peel-core/src/main/scala/org/peelframework/core/results/etl/traverser/SuiteTraverser.scala @@ -19,8 +19,7 @@ import java.io.File import java.nio.file.Path import com.typesafe.config.Config -import org.peelframework.core.results.etl.{RunState, RunStateProtocol} -import org.peelframework.core.results.etl.RunState +import org.peelframework.core.results.etl.{GenericRunState, RunStateProtocol} import resource._ import spray.json._ @@ -29,11 +28,11 @@ import spray.json._ * @param suitePath The suite path to be traversed. * @param config The application configuration. */ -class SuiteTraverser(suitePath: Path)(implicit config: Config) extends Traversable[RunState] { +class SuiteTraverser(suitePath: Path)(implicit config: Config) extends Traversable[GenericRunState] { import SuiteTraverser.loadState - override def foreach[U](f: RunState => U): Unit = for { + override def foreach[U](f: GenericRunState => U): Unit = for { dir <- suitePath.toFile.listFiles.sortBy(_.getAbsolutePath) if dir.isDirectory fil <- Option(new File(dir, "state.json")) if fil.isFile run <- loadState(fil) @@ -47,9 +46,15 @@ object SuiteTraverser { def apply(suitePath: Path)(implicit config: Config) = new SuiteTraverser(suitePath) - private def loadState(runFile: File): Option[RunState] = { + private def loadState(runFile: File): Option[GenericRunState] = { + val prefix = + s""" + |{ + | "name": "${runFile.getParentFile.getName}", + | "suiteName": "${runFile.getParentFile.getParentFile.getName}", + """.stripMargin.trim (for { src <- managed(scala.io.Source.fromFile(runFile)) - } yield src.mkString.parseJson.convertTo[RunState]).opt + } yield src.mkString.replaceFirst("""\{""", prefix).parseJson.convertTo[GenericRunState]).opt } } diff --git a/peel-extensions/src/main/scala/org/peelframework/flink/beans/experiment/FlinkExperiment.scala b/peel-extensions/src/main/scala/org/peelframework/flink/beans/experiment/FlinkExperiment.scala index 91671a49..9e0414d6 100644 --- a/peel-extensions/src/main/scala/org/peelframework/flink/beans/experiment/FlinkExperiment.scala +++ b/peel-extensions/src/main/scala/org/peelframework/flink/beans/experiment/FlinkExperiment.scala @@ -18,7 +18,6 @@ package org.peelframework.flink.beans.experiment import java.io.FileWriter import java.nio.file.{Files, Paths} -import java.lang.{System => Sys} import com.typesafe.config.Config import org.peelframework.core.beans.data.{DataSet, ExperimentOutput} import org.peelframework.core.beans.experiment.Experiment @@ -59,8 +58,6 @@ class FlinkExperiment( object FlinkExperiment { case class State( - name : String, - suiteName : String, command : String, runnerID : String, runnerName : String, @@ -70,7 +67,7 @@ object FlinkExperiment { var plnExitCode : Option[Int] = None) extends Experiment.RunState object StateProtocol extends DefaultJsonProtocol with NullOptions { - implicit val stateFormat = jsonFormat9(State) + implicit val stateFormat = jsonFormat7(State) } /** A private inner class encapsulating the logic of single run. */ @@ -87,10 +84,10 @@ object FlinkExperiment { try { scala.io.Source.fromFile(s"$home/state.json").mkString.parseJson.convertTo[State] } catch { - case e: Throwable => State(name, Sys.getProperty("app.suite.name"), command, exp.runner.beanName, exp.runner.name, exp.runner.version) + case e: Throwable => State(command, exp.runner.beanName, exp.runner.name, exp.runner.version) } } else { - State(name, Sys.getProperty("app.suite.name"), command, exp.runner.beanName, exp.runner.name, exp.runner.version) + State(command, exp.runner.beanName, exp.runner.name, exp.runner.version) } } diff --git a/peel-extensions/src/main/scala/org/peelframework/spark/beans/experiment/SparkExperiment.scala b/peel-extensions/src/main/scala/org/peelframework/spark/beans/experiment/SparkExperiment.scala index 5f311e90..088b58a3 100644 --- a/peel-extensions/src/main/scala/org/peelframework/spark/beans/experiment/SparkExperiment.scala +++ b/peel-extensions/src/main/scala/org/peelframework/spark/beans/experiment/SparkExperiment.scala @@ -16,7 +16,6 @@ package org.peelframework.spark.beans.experiment import java.io.FileWriter -import java.lang.{System => Sys} import java.nio.file._ import com.typesafe.config.Config @@ -59,8 +58,6 @@ class SparkExperiment( object SparkExperiment { case class State( - name : String, - suiteName : String, command : String, runnerID : String, runnerName : String, @@ -69,7 +66,7 @@ object SparkExperiment { var runTime : Long = 0) extends Experiment.RunState object StateProtocol extends DefaultJsonProtocol with NullOptions { - implicit val stateFormat = jsonFormat8(State) + implicit val stateFormat = jsonFormat6(State) } /** @@ -88,10 +85,10 @@ object SparkExperiment { try { scala.io.Source.fromFile(s"$home/state.json").mkString.parseJson.convertTo[State] } catch { - case e: Throwable => State(name, Sys.getProperty("app.suite.name"), command, exp.runner.beanName, exp.runner.name, exp.runner.version) + case e: Throwable => State(command, exp.runner.beanName, exp.runner.name, exp.runner.version) } } else { - State(name, Sys.getProperty("app.suite.name"), command, exp.runner.beanName, exp.runner.name, exp.runner.version) + State(command, exp.runner.beanName, exp.runner.name, exp.runner.version) } }