spark.conf

################################################################################
# Host-specific Spark configuration
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Customize Peel configuration values appearing in
#
#   https://github.com/stratosphere/peel/blob/master/peel-extensions/src/main/resources/reference.spark.conf
#
# here.
#

system {
    spark {
        config {
            # spark-env.sh entries
            env {
                HADOOP_CONF_DIR = ${system.hadoop-2.path.config}

                # enable this if you want to use spark with native libraries
                # only use if there is a hadoop version compiled with native libraries for your environment!
                # SPARK_DAEMON_JAVA_OPTS = "-Djava.library.path="${system.hadoop-2.path.home}"/lib/native"

                # 28 GiB of memory
                SPARK_EXECUTOR_MEMORY = "28672m"
                SPARK_WORKER_MEMORY = "28672m"
            }
            # spark-defaults.conf
            defaults {
                spark.master = "spark://"${runtime.hostname}":7077"

                # tmp folder for spilling data to disk (on node-local storage)
                spark.local.dir = "/data/"${user.name}"/spark/tmp"

                # 28 GiB of memory
                spark.executor.memory = "28672m"
                
                # memory of driver (e.g. to receive/gather results sets)
                spark.driver.memory = "2048m"
                
                # sets the memory limit of result sets gathered on driver to infinite
                spark.driver.maxResultSize = "0"
            }
        }
    }
}