add workload to test plan (#82)

sieve-project · Apr 3, 2022 · 38e5f1b · 38e5f1b
1 parent 79594b7
commit 38e5f1b
Show file tree

Hide file tree

Showing 43 changed files with 72 additions and 24 deletions.
diff --git a/bug_reproduction_test_plans/cass-operator-intermediate-state-1.yaml b/bug_reproduction_test_plans/cass-operator-intermediate-state-1.yaml
@@ -1,3 +1,4 @@
+workload: recreate
 actions:
 - actionType: restartController
   controllerLabel: cass-operator

diff --git a/bug_reproduction_test_plans/cass-operator-stale-state-1.yaml b/bug_reproduction_test_plans/cass-operator-stale-state-1.yaml
@@ -1,3 +1,4 @@
+workload: recreate
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/cassandra-operator-indirect-1.yaml b/bug_reproduction_test_plans/cassandra-operator-indirect-1.yaml
@@ -1 +1,2 @@
+workload: scaledown-scaleup
 actions: null
diff --git a/bug_reproduction_test_plans/cassandra-operator-indirect-2.yaml b/bug_reproduction_test_plans/cassandra-operator-indirect-2.yaml
@@ -1 +1,2 @@
+workload: scaledown-scaleup-brittle
 actions: null
diff --git a/bug_reproduction_test_plans/cassandra-operator-stale-state-1.yaml b/bug_reproduction_test_plans/cassandra-operator-stale-state-1.yaml
@@ -1,3 +1,4 @@
+workload: recreate
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/cassandra-operator-stale-state-2.yaml b/bug_reproduction_test_plans/cassandra-operator-stale-state-2.yaml
@@ -1,3 +1,4 @@
+workload: scaledown-scaleup
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/cassandra-operator-unobserved-state-1.yaml b/bug_reproduction_test_plans/cassandra-operator-unobserved-state-1.yaml
@@ -1,3 +1,4 @@
+workload: scaledown-scaleup
 actions:
 - actionType: pauseController
   pauseAt: beforeControllerRead

diff --git a/bug_reproduction_test_plans/casskop-intermediate-state-1.yaml b/bug_reproduction_test_plans/casskop-intermediate-state-1.yaml
@@ -1,3 +1,4 @@
+workload: scaledown-to-zero
 actions:
 - actionType: restartController
   controllerLabel: casskop-operator

diff --git a/bug_reproduction_test_plans/casskop-stale-state-1.yaml b/bug_reproduction_test_plans/casskop-stale-state-1.yaml
@@ -1,3 +1,4 @@
+workload: recreate
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/casskop-stale-state-2.yaml b/bug_reproduction_test_plans/casskop-stale-state-2.yaml
@@ -1,3 +1,4 @@
+workload: reducepdb
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/casskop-unobserved-state-1.yaml b/bug_reproduction_test_plans/casskop-unobserved-state-1.yaml
@@ -1,3 +1,4 @@
+workload: scaledown-to-zero
 actions:
 - actionType: pauseController
   pauseAt: beforeControllerRead

diff --git a/bug_reproduction_test_plans/elastic-operator-stale-state-1.yaml b/bug_reproduction_test_plans/elastic-operator-stale-state-1.yaml
@@ -1,3 +1,4 @@
+workload: recreate
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/elastic-operator-stale-state-2.yaml b/bug_reproduction_test_plans/elastic-operator-stale-state-2.yaml
@@ -1,3 +1,4 @@
+workload: scaledown-scaleup
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/mongodb-operator-indirect-1.yaml b/bug_reproduction_test_plans/mongodb-operator-indirect-1.yaml
@@ -1 +1,2 @@
+workload: disable-enable-shard
 actions: null
diff --git a/bug_reproduction_test_plans/mongodb-operator-indirect-2.yaml b/bug_reproduction_test_plans/mongodb-operator-indirect-2.yaml
@@ -1,3 +1,4 @@
+workload: recreate
 actions:
 - actionType: pauseController
   pauseAt: afterControllerWrite

diff --git a/bug_reproduction_test_plans/mongodb-operator-indirect-3.yaml b/bug_reproduction_test_plans/mongodb-operator-indirect-3.yaml
@@ -1 +1,2 @@
+workload: disable-enable-shard-brittle
 actions: null
diff --git a/bug_reproduction_test_plans/mongodb-operator-intermediate-state-1.yaml b/bug_reproduction_test_plans/mongodb-operator-intermediate-state-1.yaml
@@ -1,3 +1,4 @@
+workload: disable-enable-shard
 actions:
 - actionType: restartController
   controllerLabel: mongodb-operator

diff --git a/bug_reproduction_test_plans/mongodb-operator-intermediate-state-2.yaml b/bug_reproduction_test_plans/mongodb-operator-intermediate-state-2.yaml
@@ -1,3 +1,4 @@
+workload: run-cert-manager
 actions:
 - actionType: restartController
   controllerLabel: mongodb-operator

diff --git a/bug_reproduction_test_plans/mongodb-operator-stale-state-1.yaml b/bug_reproduction_test_plans/mongodb-operator-stale-state-1.yaml
@@ -1,3 +1,4 @@
+workload: recreate
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/mongodb-operator-stale-state-2.yaml b/bug_reproduction_test_plans/mongodb-operator-stale-state-2.yaml
@@ -1,3 +1,4 @@
+workload: disable-enable-shard
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/mongodb-operator-stale-state-3.yaml b/bug_reproduction_test_plans/mongodb-operator-stale-state-3.yaml
@@ -1,3 +1,4 @@
+workload: disable-enable-arbiter
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/mongodb-operator-unobserved-state-1.yaml b/bug_reproduction_test_plans/mongodb-operator-unobserved-state-1.yaml
@@ -1,3 +1,4 @@
+workload: disable-enable-arbiter
 actions:
 - actionType: pauseController
   pauseAt: beforeControllerRead

diff --git a/bug_reproduction_test_plans/nifikop-intermediate-state-1.yaml b/bug_reproduction_test_plans/nifikop-intermediate-state-1.yaml
@@ -1,3 +1,4 @@
+workload: change-config
 actions:
 - actionType: restartController
   controllerLabel: nifikop-operator

diff --git a/bug_reproduction_test_plans/rabbitmq-operator-intermediate-state-1.yaml b/bug_reproduction_test_plans/rabbitmq-operator-intermediate-state-1.yaml
@@ -1,3 +1,4 @@
+workload: resize-pvc
 actions:
 - actionType: restartController
   controllerLabel: rabbitmq-operator

diff --git a/bug_reproduction_test_plans/rabbitmq-operator-stale-state-1.yaml b/bug_reproduction_test_plans/rabbitmq-operator-stale-state-1.yaml
@@ -1,3 +1,4 @@
+workload: recreate
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/rabbitmq-operator-stale-state-2.yaml b/bug_reproduction_test_plans/rabbitmq-operator-stale-state-2.yaml
@@ -1,3 +1,4 @@
+workload: resize-pvc
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/rabbitmq-operator-unobserved-state-1.yaml b/bug_reproduction_test_plans/rabbitmq-operator-unobserved-state-1.yaml
@@ -1,3 +1,4 @@
+workload: scaleup-scaledown
 actions:
 - actionType: pauseController
   pauseAt: beforeControllerRead

diff --git a/bug_reproduction_test_plans/xtradb-operator-intermediate-state-1.yaml b/bug_reproduction_test_plans/xtradb-operator-intermediate-state-1.yaml
@@ -1,3 +1,4 @@
+workload: disable-enable-proxysql
 actions:
 - actionType: restartController
   controllerLabel: xtradb-operator

diff --git a/bug_reproduction_test_plans/xtradb-operator-intermediate-state-2.yaml b/bug_reproduction_test_plans/xtradb-operator-intermediate-state-2.yaml
@@ -1,3 +1,4 @@
+workload: run-cert-manager
 actions:
 - actionType: restartController
   controllerLabel: xtradb-operator

diff --git a/bug_reproduction_test_plans/xtradb-operator-stale-state-1.yaml b/bug_reproduction_test_plans/xtradb-operator-stale-state-1.yaml
@@ -1,3 +1,4 @@
+workload: recreate
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/xtradb-operator-stale-state-2.yaml b/bug_reproduction_test_plans/xtradb-operator-stale-state-2.yaml
@@ -1,3 +1,4 @@
+workload: disable-enable-haproxy
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/xtradb-operator-stale-state-3.yaml b/bug_reproduction_test_plans/xtradb-operator-stale-state-3.yaml
@@ -1,3 +1,4 @@
+workload: disable-enable-proxysql
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/xtradb-operator-unobserved-state-1.yaml b/bug_reproduction_test_plans/xtradb-operator-unobserved-state-1.yaml
@@ -1,3 +1,4 @@
+workload: scaleup-scaledown
 actions:
 - actionType: pauseController
   pauseAt: beforeControllerRead

diff --git a/bug_reproduction_test_plans/yugabyte-operator-indirect-1.yaml b/bug_reproduction_test_plans/yugabyte-operator-indirect-1.yaml
@@ -1 +1,2 @@
+workload: disable-enable-tuiport
 actions: null
diff --git a/bug_reproduction_test_plans/yugabyte-operator-indirect-2.yaml b/bug_reproduction_test_plans/yugabyte-operator-indirect-2.yaml
@@ -1 +1,2 @@
+workload: disable-enable-tls
 actions: null
diff --git a/bug_reproduction_test_plans/yugabyte-operator-stale-state-1.yaml b/bug_reproduction_test_plans/yugabyte-operator-stale-state-1.yaml
@@ -1,3 +1,4 @@
+workload: disable-enable-tls
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/yugabyte-operator-stale-state-2.yaml b/bug_reproduction_test_plans/yugabyte-operator-stale-state-2.yaml
@@ -1,3 +1,4 @@
+workload: disable-enable-tuiport
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/yugabyte-operator-unobserved-state-1.yaml b/bug_reproduction_test_plans/yugabyte-operator-unobserved-state-1.yaml
@@ -1,3 +1,4 @@
+workload: scaleup-scaledown-tserver
 actions:
 - actionType: pauseController
   pauseAt: beforeControllerRead

diff --git a/bug_reproduction_test_plans/zookeeper-operator-indirect-1.yaml b/bug_reproduction_test_plans/zookeeper-operator-indirect-1.yaml
@@ -1,3 +1,4 @@
+workload: recreate
 actions:
 - actionType: pauseController
   pauseAt: afterControllerWrite

diff --git a/bug_reproduction_test_plans/zookeeper-operator-stale-state-1.yaml b/bug_reproduction_test_plans/zookeeper-operator-stale-state-1.yaml
@@ -1,3 +1,4 @@
+workload: recreate
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/bug_reproduction_test_plans/zookeeper-operator-stale-state-2.yaml b/bug_reproduction_test_plans/zookeeper-operator-stale-state-2.yaml
@@ -1,3 +1,4 @@
+workload: scaledown-scaleup
 actions:
 - actionType: pauseAPIServer
   apiServerName: kind-control-plane3

diff --git a/reproduce_bugs.py b/reproduce_bugs.py
@@ -195,15 +195,11 @@ def reproduce_single_bug(operator, bug, docker, phase):
         before_reproduce()
     test = reprod_map[operator][bug][0]
     config = os.path.join("bug_reproduction_test_plans", reprod_map[operator][bug][1])
-    sieve_cmd = (
-        "python3 sieve.py -p %s -s test -m test -t %s -c %s -d %s --phase=%s"
-        % (
-            operator,
-            test,
-            config,
-            docker,
-            phase,
-        )
+    sieve_cmd = "python3 sieve.py -p %s -c %s -d %s --phase=%s" % (
+        operator,
+        config,
+        docker,
+        phase,
     )
     cprint(sieve_cmd, bcolors.OKGREEN)
     os.system(sieve_cmd)

diff --git a/sieve.py b/sieve.py
@@ -70,7 +70,9 @@ def save_run_result(
                         + test_result.history_errors,
                         "no_exception": test_result.no_exception,
                         "exception_message": test_result.exception_message,
-                        "test_config_content": open(test_config).read() if mode == sieve_modes.TEST else "",
+                        "test_config_content": open(test_config).read()
+                        if mode == sieve_modes.TEST
+                        else "",
                         "host": socket.gethostname(),
                     }
                 }
@@ -592,6 +594,11 @@ def generate_vanilla_config(vanilla_config):
     yaml.dump(vanilla_config_map, open(vanilla_config, "w"), sort_keys=False)
 
 
+def get_test_workload_from_test_plan(test_plan_file):
+    test_plan = yaml.safe_load(open(test_plan_file))
+    return test_plan["workload"]
+
+
 def run(
     project,
     test,
@@ -608,6 +615,10 @@ def run(
     num_apiservers = 1
     num_workers = 2
     use_csi_driver = False
+    if test is None:
+        assert stage == sieve_stages.TEST and mode == sieve_modes.TEST
+        test = get_test_workload_from_test_plan(config)
+        print("get test workload {} from test plan".format(test))
     if test in controller_config.test_setting:
         if "num_apiservers" in controller_config.test_setting[test]:
             num_apiservers = controller_config.test_setting[test]["num_apiservers"]
@@ -707,11 +718,18 @@ def run_batch(project, test, dir, mode, stage, docker):
     parser.add_option(
         "-l", "--log", dest="log", help="save to LOG", metavar="LOG", default="log"
     )
+    parser.add_option(
+        "-s",
+        "--stage",
+        dest="stage",
+        help="STAGE: learn, test",
+        default=sieve_stages.TEST,
+    )
     parser.add_option(
         "-m",
         "--mode",
         dest="mode",
-        help="test MODE: vanilla, stale-state, unobserved-state, intermediate-state",
+        help="MODE: vanilla, test, learn-once, learn-twice",
         metavar="MODE",
     )
     parser.add_option(
@@ -736,12 +754,6 @@ def run_batch(project, test, dir, mode, stage, docker):
         metavar="PHASE",
         default="all",
     )
-    parser.add_option(
-        "-s",
-        "--stage",
-        dest="stage",
-        help="STAGE: learn, test",
-    )
     parser.add_option(
         "-r",
         "--rate_limiter",
@@ -756,12 +768,7 @@ def run_batch(project, test, dir, mode, stage, docker):
     if options.project is None:
         parser.error("parameter project required")
 
-    if options.test is None:
-        parser.error("parameter test required")
-
-    if options.stage is None:
-        parser.error("parameter stage required")
-    elif options.stage not in [sieve_stages.LEARN, sieve_stages.TEST]:
+    if options.stage not in [sieve_stages.LEARN, sieve_stages.TEST]:
         parser.error("invalid stage option: %s" % options.stage)
 
     if options.stage == sieve_stages.LEARN:
@@ -776,7 +783,7 @@ def run_batch(project, test, dir, mode, stage, docker):
 
     if options.stage == sieve_stages.TEST:
         if options.mode is None:
-            parser.error("parameter mode required in test stage")
+            options.mode = sieve_modes.TEST
         elif options.mode not in [sieve_modes.TEST, sieve_modes.VANILLA]:
             parser.error("invalid test mode option: %s" % options.mode)
         if options.mode == sieve_modes.VANILLA:
@@ -785,6 +792,10 @@ def run_batch(project, test, dir, mode, stage, docker):
             if options.config is None:
                 parser.error("parameter config required in test stage")
 
+    if options.test is None:
+        if options.stage != sieve_stages.TEST or options.mode != sieve_stages.TEST:
+            parser.error("parameter test required")
+
     if options.phase not in [
         "all",
         "setup",
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		workload: scaledown-scaleup-brittle
		actions: null
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		workload: disable-enable-shard-brittle
		actions: null