Recreate test appdomain between each benchmark

src/NBench.PerformanceCounters/Metrics/PerformanceCounterBenchmarkSetting.cs

@@ -10,7 +10,7 @@ namespace NBench.PerformanceCounters.Metrics
     /// <summary>
     /// <see cref="IBenchmarkSetting"/> used to instrument performance counters inside a <see cref="Benchmark"/>
     /// </summary>
-    public sealed class PerformanceCounterBenchmarkSetting : IBenchmarkSetting, IEquatable<PerformanceCounterBenchmarkSetting>
+    public sealed class PerformanceCounterBenchmarkSetting : MarshalByRefObject, IBenchmarkSetting, IEquatable<PerformanceCounterBenchmarkSetting>
     {
         public PerformanceCounterBenchmarkSetting(PerformanceCounterMetricName performanceCounterMetric, AssertionType assertionType, Assertion assertion)
         {

src/NBench/Collection/MetricsCollectorSelector.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
-// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
-
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using System;
 using System.Collections.Generic;
 using NBench.Metrics;
 using NBench.Sdk;
@@ -12,7 +13,7 @@ namespace NBench.Collection
     ///     Strategy pattern that's used to determine how to pick the appropriate
     ///     <see cref="MetricCollector" /> based on platform dependencies and user-preferences.
     /// </summary>
-    public abstract class MetricsCollectorSelector
+    public abstract class MetricsCollectorSelector: MarshalByRefObject
     {
         protected MetricsCollectorSelector(MetricName name) : this(name, SysInfo.Instance)
         {

src/NBench/Metrics/Counters/CounterBenchmarkSetting.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
-// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
-
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using System;
 using System.Collections.Generic;
 using System.Diagnostics.Contracts;
 using NBench.Sdk;
@@ -11,7 +12,7 @@ namespace NBench.Metrics.Counters
     /// Used inside a <see cref="BenchmarkSettings"/> class to indiciate that a specific <see cref="Counter"/>
     /// needs to be recorded and tested against <see cref="Assertion"/>.
     /// </summary>
-    public sealed class CounterBenchmarkSetting : IBenchmarkSetting
+    public sealed class CounterBenchmarkSetting : MarshalByRefObject, IBenchmarkSetting
     {
         public CounterBenchmarkSetting(string counterName, AssertionType assertionType, Assertion assertion)
         {

src/NBench/Metrics/Counters/CreateCounterBenchmarkSetting.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
-// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
-
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using System;
 using NBench.Collection;
 using NBench.Collection.Counters;
 using NBench.Sdk;
@@ -11,7 +12,7 @@ namespace NBench.Metrics.Counters
     /// <summary>
     /// Special internal class needed to pass in a <see cref="AtomicCounter"/> to a <see cref="CounterSelector"/>.
     /// </summary>
-    internal sealed class CreateCounterBenchmarkSetting : IBenchmarkSetting {
+    internal sealed class CreateCounterBenchmarkSetting : MarshalByRefObject, IBenchmarkSetting {
         public CreateCounterBenchmarkSetting(CounterBenchmarkSetting benchmarkSetting, AtomicCounter counter)
         {
             BenchmarkSetting = benchmarkSetting;

src/NBench/Metrics/GarbageCollection/GcBenchmarkSetting.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
-// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
-
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using System;
 using System.Collections.Generic;
 using NBench.Sdk;
 
@@ -10,7 +11,7 @@ namespace NBench.Metrics.GarbageCollection
     /// Used inside a <see cref="BenchmarkSettings"/> class to indiciate that a specific <see cref="GcMetric"/>
     /// for a <see cref="GcGeneration"/> needs to be recorded and tested against <see cref="Assertion"/>.
     /// </summary>
-    public sealed class GcBenchmarkSetting : IBenchmarkSetting
+    public sealed class GcBenchmarkSetting : MarshalByRefObject, IBenchmarkSetting
     {
         public GcBenchmarkSetting(GcMetric metric, GcGeneration generation, AssertionType assertionType, Assertion assertion)
         {

src/NBench/Metrics/Memory/MemoryBenchmarkSetting.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
-// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
-
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using System;
 using System.Collections.Generic;
 using NBench.Sdk;
 
@@ -10,7 +11,7 @@ namespace NBench.Metrics.Memory
     /// Used inside a <see cref="BenchmarkSettings"/> class to indiciate that a specific <see cref="MemoryMetric"/>
     /// needs to be recorded and tested against <see cref="Assertion"/>.
     /// </summary>
-    public sealed class MemoryBenchmarkSetting : IBenchmarkSetting
+    public sealed class MemoryBenchmarkSetting : MarshalByRefObject, IBenchmarkSetting
     {
         public MemoryBenchmarkSetting(MemoryMetric metric, Assertion assertion)
         {

src/NBench/Metrics/MetricName.cs

@@ -8,7 +8,7 @@ namespace NBench.Metrics
     /// <summary>
     /// Used to uniquely represent the name of a metric in a manner-agnostic way.
     /// </summary>
-    public abstract class MetricName : IEquatable<MetricName>
+    public abstract class MetricName : MarshalByRefObject, IEquatable<MetricName>
     {
         public abstract bool Equals(MetricName other);
 

src/NBench/Metrics/Timing/TimingBenchmarkSetting.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
-// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
-
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using System;
 using NBench.Sdk;
 
 namespace NBench.Metrics.Timing
@@ -9,7 +10,7 @@ namespace NBench.Metrics.Timing
     /// <see cref="IBenchmarkSetting"/> implementation for <see cref="TimingMeasurementAttribute"/>
     /// and other derived versions.
     /// </summary>
-    public sealed class TimingBenchmarkSetting : IBenchmarkSetting
+    public sealed class TimingBenchmarkSetting : MarshalByRefObject, IBenchmarkSetting
     {
         public TimingBenchmarkSetting(TimingMetricName metricName, Assertion assertion)
         {

src/NBench/NBench.csproj

@@ -127,6 +127,7 @@
     <Compile Include="Reporting\Targets\MarkdownBenchmarkOutput.cs" />
     <Compile Include="Sdk\Assertion.cs" />
     <Compile Include="Sdk\AssertionResult.cs" />
+    <Compile Include="Sdk\BenchmarkRunner.cs" />
     <Compile Include="Sdk\DefaultBenchmarkAssertionRunner.cs" />
     <Compile Include="Sdk\AssertionType.cs" />
     <Compile Include="BenchmarkContext.cs" />

src/NBench/Reporting/AggregateMetrics.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
-// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
-
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.Diagnostics.Contracts;
@@ -12,7 +13,8 @@ namespace NBench.Reporting
 {
     /// <summary>
     /// Aggregated metrics accumulated for a single metric across an entire <see cref="Benchmark"/>
-    /// </summary>
+    /// </summary>
+    [Serializable]
     public struct AggregateMetrics
     {
         private static IReadOnlyList<MetricRunReport> GetSafeRuns(MetricName name, string unit)

src/NBench/Reporting/BenchmarkFinalResults.cs

@@ -1,14 +1,16 @@
 // Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
-// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
-
+// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.
+
+using System;
 using System.Collections.Generic;
 using NBench.Sdk;
 
 namespace NBench.Reporting
 {
     /// <summary>
     /// <see cref="BenchmarkResults"/> with <see cref="Assertion"/> data provided.
-    /// </summary>
+    /// </summary>
+    [Serializable]
     public class BenchmarkFinalResults
     {
         public BenchmarkFinalResults(BenchmarkResults data, IReadOnlyList<AssertionResult> assertionResults)

src/NBench/Reporting/BenchmarkResults.cs

@@ -12,7 +12,8 @@ namespace NBench.Reporting
 {
     /// <summary>
     ///     The cumulative results for an entire <see cref="Benchmark" />
-    /// </summary>
+    /// </summary>
+    [Serializable]
     public class BenchmarkResults
     {
         public BenchmarkResults(string typeName, BenchmarkSettings settings, IReadOnlyList<BenchmarkRunReport> runs)

src/NBench/Reporting/BenchmarkRunReport.cs

@@ -10,7 +10,8 @@ namespace NBench.Reporting
 {
     /// <summary>
     ///     Compiled statistics for each <see cref="BenchmarkRun" />
-    /// </summary>
+    /// </summary>
+    [Serializable]
     public struct BenchmarkRunReport
     {
         public BenchmarkRunReport(TimeSpan elapsed, IEnumerable<MetricRunReport> metrics, IReadOnlyList<Exception> exceptions)

src/NBench/Reporting/BenchmarkStat.cs

@@ -10,7 +10,8 @@ namespace NBench.Reporting
 {
     /// <summary>
     /// Represents aggregate statics for a benchmark across multiple runs
-    /// </summary>
+    /// </summary>
+    [Serializable]
     public struct BenchmarkStat
     {
         public static readonly double[] SafeValues = {0.0d};

src/NBench/Reporting/CompositeBenchmarkOutput.cs

@@ -9,8 +9,8 @@ namespace NBench.Reporting
 {
     /// <summary>
     /// Composition of multiple <see cref="IBenchmarkOutput"/> instances being run in parallel.
-    /// </summary>
-    public class CompositeBenchmarkOutput : IBenchmarkOutput
+    /// </summary>
+    public class CompositeBenchmarkOutput : MarshalByRefObject, IBenchmarkOutput
     {
         private readonly IReadOnlyList<IBenchmarkOutput> _outputs;
 

src/NBench/Reporting/MetricRunReport.cs

@@ -10,7 +10,8 @@ namespace NBench.Reporting
 {
     /// <summary>
     /// All of the compiled values from one <see cref="BenchmarkRun"/> for a given <see cref="MeasureBucket"/>
-    /// </summary>
+    /// </summary>
+    [Serializable]
     public struct MetricRunReport
     {
         public MetricRunReport(MetricName name, string unit, double metricReading, long ticks)

src/NBench/Reporting/Targets/ConsoleBenchmarkOutput.cs

@@ -8,7 +8,7 @@ namespace NBench.Reporting.Targets
     /// <summary>
     /// Output writer to the console for NBench
     /// </summary>
-    public class ConsoleBenchmarkOutput : IBenchmarkOutput
+    public class ConsoleBenchmarkOutput : MarshalByRefObject, IBenchmarkOutput
     {
         public void WriteLine(string message)
         {

src/NBench/Reporting/Targets/MarkdownBenchmarkOutput.cs

@@ -18,7 +18,7 @@ namespace NBench.Reporting.Targets
     ///     completed benchmark to a markdown file. Uses <see cref="FileNameGenerator" />
     ///     to generate a file name unique to each test AND the time it was run.
     /// </summary>
-    public class MarkdownBenchmarkOutput : IBenchmarkOutput
+    public class MarkdownBenchmarkOutput : MarshalByRefObject, IBenchmarkOutput
     {
         public const string MarkdownFileExtension = ".md";
         public const int MaxColumnSize = 18;

src/NBench/Sdk/AssertionResult.cs

@@ -7,6 +7,7 @@
 
 namespace NBench.Sdk
 {
+    [Serializable]
     public struct AssertionResult
     {
         public AssertionResult(MetricName metricName, string message, bool passed)

src/NBench/Sdk/BenchMarkRunner.cs

@@ -0,0 +1,40 @@
+using System;
+using System.Linq;
+using NBench.Reporting;
+using NBench.Sdk.Compiler;
+
+namespace NBench.Sdk
+{
+    [Serializable]
+    internal class BenchmarkRunnerResult
+    {
+        public bool AllTestsPassed { get; set; }
+    }
+
+    internal class BenchmarkRunner : MarshalByRefObject
+    {
+        private readonly IBenchmarkOutput _output;
+
+        public BenchmarkRunner(IBenchmarkOutput output)
+        {
+            _output = output;
+        }
+
+        public BenchmarkRunnerResult Run(string testFile, string benchmarkName)
+        {
+            var discovery = new ReflectionDiscovery(_output);
+
+            var assembly = AssemblyRuntimeLoader.LoadAssembly(testFile);
+            var benchmarks = discovery.FindBenchmarks(assembly);
+
+            var benchmark = benchmarks.FirstOrDefault(b => b.BenchmarkName == benchmarkName);
+            benchmark.Run();
+            benchmark.Finish();
+
+            return new BenchmarkRunnerResult
+            {
+                AllTestsPassed = benchmark.AllAssertsPassed,
+            };
+        }
+    }
+}

src/NBench/Sdk/BenchmarkSettings.cs

@@ -15,7 +15,8 @@ namespace NBench.Sdk
 {
     /// <summary>
     ///     Settings for how a particular <see cref="Benchmark" /> should be run and executed.
-    /// </summary>
+    /// </summary>
+    [Serializable]
     public class BenchmarkSettings
     {
         internal class BenchmarkEqualityComparer : IEqualityComparer<IBenchmarkSetting>

src/NBench/Sdk/TestRunner.cs

@@ -65,19 +65,8 @@ public static TestRunner CreateRunner(AppDomain domain, TestPackage package)
         /// <remarks>Creates a new AppDomain and executes the tests.</remarks>
         public static TestRunnerResult Run(TestPackage package)
         {
-            Contract.Requires(package != null);
-            // create the test app domain
-            var testDomain = DomainManager.CreateDomain(package);
-
-            try
-            {
-                var runner = TestRunner.CreateRunner(testDomain, package);
-                return runner.Execute();
-            }
-            finally
-            {
-                DomainManager.UnloadDomain(testDomain);
-            }
+            var runner = new TestRunner(package);
+            return runner.Execute();
         }
 
         /// <summary>
@@ -152,12 +141,22 @@ public TestRunnerResult Execute()
                         // verify if the benchmark should be included/excluded from the list of benchmarks to be run
                         if (_package.ShouldRunBenchmark(benchmark.BenchmarkName))
                         {
+                            var testDomain = DomainManager.CreateDomain(_package);
+                            var benchmarkRunnerType = typeof(BenchmarkRunner);
+                            var benchMarkRunner = testDomain.CreateInstanceAndUnwrap(benchmarkRunnerType.Assembly.FullName, benchmarkRunnerType.FullName, false, 0, null, new object[] { output }, null, null) as BenchmarkRunner;
+
                             output.WriteLine($"------------ STARTING {benchmark.BenchmarkName} ---------- ");
-                            benchmark.Run();
-                            benchmark.Finish();
+                            try
+                            {
+                                var benchMarkResult = benchMarkRunner.Run(testFile, benchmark.BenchmarkName);
+                                // if one assert fails, all fail
+                                result.AllTestsPassed = result.AllTestsPassed && benchMarkResult.AllTestsPassed;
+                            }
+                            finally
+                            {
+                                DomainManager.UnloadDomain(testDomain);
+                            }
 
-                            // if one assert fails, all fail
-                            result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed;
                             output.WriteLine($"------------ FINISHED {benchmark.BenchmarkName} ---------- ");
                             result.ExecutedTestsCount = result.ExecutedTestsCount + 1;
                         }

src/NBench/Tracing/NoOpBenchmarkTrace.cs

@@ -5,7 +5,7 @@ namespace NBench.Tracing
     /// <summary>
     /// Default no-op implementation of <see cref="IBenchmarkTrace"/>. Does nothing.
     /// </summary>
-    internal sealed class NoOpBenchmarkTrace : IBenchmarkTrace
+    internal sealed class NoOpBenchmarkTrace : MarshalByRefObject, IBenchmarkTrace
     {
         public static readonly NoOpBenchmarkTrace Instance = new NoOpBenchmarkTrace();