-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathtest_spec_test.py
147 lines (109 loc) · 5.11 KB
/
test_spec_test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
import pytest
from unittest.mock import patch
from testbeds.swebench.constants import ResolvedStatus
from testbeds.swebench.test_spec import TestSpec
from testbeds.schema import SWEbenchInstance
from testbeds.swebench.utils import load_swebench_instance
@pytest.fixture
def sample_instance():
return SWEbenchInstance(
instance_id="django__django-12345",
repo="django/django",
version="4.2",
base_commit="abcdef123456",
patch="--- a/some_file.py\n+++ b/some_file.py\n@@ -1,1 +1,2 @@\n+# Patch content",
test_patch="--- a/tests/test_file.py\n+++ b/tests/test_file.py\n@@ -1,1 +1,2 @@\n+# New test",
problem_statement="Fix the bug in the authentication system",
created_at="2023-01-01T00:00:00Z",
fail_to_pass=["test_case_1"],
pass_to_pass=["test_case_2"],
environment_setup_commit="fedcba987654",
)
def test_test_spec_from_instance(sample_instance):
test_spec = TestSpec.from_instance(sample_instance)
assert test_spec.instance_id == "django__django-12345"
assert test_spec.repo == "django/django"
assert test_spec.version == "4.2"
assert test_spec.base_commit == "abcdef123456"
assert test_spec.test_patch == sample_instance.test_patch
assert test_spec.fail_to_pass == ["test_case_1"]
assert test_spec.pass_to_pass == ["test_case_2"]
assert test_spec.arch in ["x86_64", "arm64"]
def test_eval_script_property(sample_instance):
test_spec = TestSpec.from_instance(sample_instance)
eval_script = test_spec.eval_script
assert "#!/bin/bash" in eval_script
assert "set -uxo pipefail" in eval_script
assert "git checkout" in eval_script
assert "git apply -v" in eval_script
def test_install_repo_script_property(sample_instance):
test_spec = TestSpec.from_instance(sample_instance)
install_script = test_spec.install_repo_script
assert "#!/bin/bash" in install_script
assert "set -euxo pipefail" in install_script
assert "git clone" in install_script
assert "conda activate" in install_script
def test_base_image_key_property(sample_instance):
test_spec = TestSpec.from_instance(sample_instance)
assert test_spec.base_image_key.startswith("sweb.base.")
assert test_spec.base_image_key.endswith(":latest")
def test_env_image_key_property(sample_instance):
test_spec = TestSpec.from_instance(sample_instance)
assert test_spec.env_image_key.startswith("sweb.env.")
assert len(test_spec.env_image_key.split(".")) == 4
def test_instance_image_key_property(sample_instance):
test_spec = TestSpec.from_instance(sample_instance)
assert (
test_spec.instance_image_key
== f"sweb.eval.{test_spec.arch}.{test_spec.instance_id}:latest"
)
def test_get_instance_container_name(sample_instance):
test_spec = TestSpec.from_instance(sample_instance)
assert (
test_spec.get_instance_container_name() == f"sweb.eval.{test_spec.instance_id}"
)
assert (
test_spec.get_instance_container_name("run1")
== f"sweb.eval.{test_spec.instance_id}.run1"
)
def test_platform_property(sample_instance):
test_spec = TestSpec.from_instance(sample_instance)
assert test_spec.platform in ["linux/x86_64", "linux/arm64/v8"]
def test_patch_commands(sample_instance):
test_spec = TestSpec.from_instance(sample_instance)
patch_commands = test_spec.patch_commands("/path/to/patch")
assert "git apply -v /path/to/patch" in patch_commands
assert "patch --batch --fuzz=5 -p1 -i /path/to/patch" in " ".join(patch_commands)
def test_get_test_directives(sample_instance):
test_spec = TestSpec.from_instance(sample_instance)
directives = test_spec.get_test_patch_files()
assert directives == [] # or update the implementation to return ["test_file"]
def test_django_eval():
instance_id = "django__django-16041"
instance = load_swebench_instance(instance_id)
test_spec = TestSpec.from_instance(instance)
with open("tests/data/django_output_3.txt", "r") as f:
content = f.read()
result = test_spec.get_pred_report(content)
assert result.status == ResolvedStatus.NO
assert result.fail_to_pass.failure == [
"test_empty_permitted_ignored_empty_form (forms_tests.tests.test_formsets.FormsFormsetTestCase)",
"test_empty_permitted_ignored_empty_form (forms_tests.tests.test_formsets.Jinja2FormsFormsetTestCase)",
]
# This text is used on two different tests but will just show once in the output
assert result.pass_to_pass.failure == [
"The extra argument works when the formset is pre-filled with initial"
]
@pytest.mark.skip
def test_pytest_eval():
instance_id = "pytest-dev__pytest-7373"
instance = load_swebench_instance(instance_id)
test_spec = TestSpec.from_instance(instance)
with open("tests/data/pytest_output_5.txt", "r") as f:
content = f.read()
result = test_spec.get_pred_report(content)
assert len(result.fail_to_pass.success) == 1
assert len(result.fail_to_pass.failure) == 0
assert len(result.pass_to_pass.success) == 81
assert len(result.pass_to_pass.failure) == 0
assert result.status == ResolvedStatus.FULL