Refactor and document sieve testing script (#111)

* Refactor Sieve testing script

* Remove evaluation_sanity_check

* Fix check_test_plan_gen.py

* Document Sieve script

* Fix parallel testing scripts

* Fix bug in sieve client and sieve analyze for supporting REST read

check_test_plan_gen.py

@@ -1,9 +1,26 @@
-from evaluation_sanity_check import check, generate
-import sys
+import glob
+import os
 
-# current_dir = "log"
-# previous_dir = sys.argv[1]
 
-# generate.generate_test_plan_stat()
-# check.check_massive_testing_results(current_dir, previous_dir)
-check.check_bug_reproduction_test_plans()
+def check_bug_reproduction_test_plans():
+    gen_configs = glob.glob(
+        os.path.join(
+            "sieve_learn_results/*/*/learn/*/*.yaml",
+        )
+    )
+
+    reprod_configs = glob.glob("bug_reproduction_test_plans/*.yaml")
+
+    for reprod_config in reprod_configs:
+        if "indirect" in reprod_config:
+            continue
+        found = False
+        for gen_config in gen_configs:
+            if open(reprod_config).read() == open(gen_config).read():
+                print(reprod_config + " <= " + gen_config)
+                found = True
+        if not found:
+            print("\033[91m" + reprod_config + " not found\033[0m")
+
+
+check_bug_reproduction_test_plans()

parallel_testing/combine_json.py

@@ -30,10 +30,9 @@ def merge(result, patch):
     generated_test_plans = glob.glob(
         os.path.join(
             result_folder,
-            "log",
             controller,
             "*",
-            "generate-oracle/learn.yaml",
+            "learn",
             "*",
             "*-test-plan-*.yaml",
         )

parallel_testing/gen_commands.py

@@ -2,32 +2,32 @@
 import os
 import argparse
 
-modes = ["intermediate-state", "unobserved-state", "stale-state"]
+patterns = ["intermediate-state", "unobserved-state", "stale-state"]
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate testcase commands into a file."
     )
     parser.add_argument(
-        "-d",
-        dest="docker",
-        help="Docker account",
+        "-r",
+        dest="registry",
+        help="Container registry",
         default="ghcr.io/sieve-project/action",
     )
     parser.add_argument(
         "-o", dest="output", help="Output file name", default="commands.txt"
     )
     parser.add_argument("-c", dest="controllers", help="Controllers to test", nargs="+")
-    parser.add_argument("-m", dest="modes", help="Modes to test", nargs="+")
+    parser.add_argument(
+        "--pattern", dest="patterns", help="Patterns to test", nargs="+"
+    )
     args = parser.parse_args()
 
     if args.controllers is None:
-        controllers = os.listdir("../log")
-    else:
-        controllers = args.controllers
+        args.controllers = os.listdir("../log")
 
-    if args.modes is not None:
-        modes = args.modes
+    if args.patterns is None:
+        args.patterns = patterns
 
     with open(args.output, "w") as command_file, open(
         "pull-commands.txt", "w"
@@ -36,41 +36,30 @@
         pull_command_file.write(
             "docker pull {}/node:v1.18.9-test\n".format(args.docker)
         )
-        pull_command_file.write(
-            "docker pull {}/node:v1.18.9-vanilla\n".format(args.docker)
-        )
-        for controller in controllers:
+        for controller in args.controllers:
             pull_command_file.write(
                 "docker pull {}/{}:test\n".format(args.docker, controller)
             )
-            pull_command_file.write(
-                "docker pull {}/{}:vanilla\n".format(args.docker, controller)
-            )
-            for mode in modes:
-                for testcase in os.listdir(os.path.join("../log", controller)):
-                    if mode == "vanilla":
-                        command_file.write(
-                            "python3 sieve.py -m vanilla -c {} -w {} -r {}\n".format(
-                                controller, testcase, args.docker
-                            )
+            for pattern in args.patterns:
+                for test_workload in os.listdir(
+                    os.path.join("../sieve_learn_results", controller)
+                ):
+                    test_plans = glob.glob(
+                        os.path.join(
+                            os.path.abspath("../sieve_learn_results"),
+                            controller,
+                            test_workload,
+                            "learn",
+                            pattern,
+                            "*.yaml",
                         )
-                    else:
-                        configs = glob.glob(
-                            os.path.join(
-                                os.path.abspath("../log"),
+                    )
+                    for test_plan in test_plans:
+                        command_file.write(
+                            "python3 sieve.py -m test -c {} -w {} -p {} -r {}\n".format(
                                 controller,
-                                testcase,
-                                "generate-oracle/learn.yaml",
-                                mode,
-                                "*.yaml",
+                                test_workload,
+                                test_plan,
+                                args.docker,
                             )
                         )
-                        for config in configs:
-                            command_file.write(
-                                "python3 sieve.py -m test -c {} -w {} -p {} -r {}\n".format(
-                                    controller,
-                                    testcase,
-                                    config,
-                                    args.docker,
-                                )
-                            )