Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move expected tracer.events from test code to golden files #133

Merged
merged 15 commits into from
Oct 8, 2021
Merged

Move expected tracer.events from test code to golden files #133

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
24a7716
Move assertions to golden files
victorjzsun Aug 26, 2021
8a55728
Implement new fixture for asserting tracer.events
victorjzsun Sep 16, 2021
ad6d847
Add mocked_responses to all tests
victorjzsun Sep 17, 2021
d099926
Add class name to event
victorjzsun Sep 17, 2021
6f05635
Remove import test due to repr naming
victorjzsun Oct 4, 2021
a9a4b02
Formatting
victorjzsun Oct 4, 2021
575227c
Use both fixtures
victorjzsun Oct 4, 2021
3d823f8
Remove unnecessary fixture
victorjzsun Oct 4, 2021
c38c778
Remove tracer.events assertions
victorjzsun Oct 4, 2021
fdc4f7a
Fix test
victorjzsun Oct 4, 2021
46dbcb4
Remove extra assert
victorjzsun Oct 4, 2021
8effcbf
Renaming
victorjzsun Oct 5, 2021
be40db8
Reformatting as per comments
victorjzsun Oct 6, 2021
d7c0d1e
Lint
victorjzsun Oct 6, 2021
0e75b80
Add docs
victorjzsun Oct 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cyberbrain/basis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ class Event:

@staticmethod
def value_serializer(inst: type, field: attr.Attribute, value: Any):
if field is None:
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we need this?

Copy link
Collaborator Author

@victorjzsun victorjzsun Oct 6, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not particularly needed for this PR but if you use it in attr.asdict without the filter removing sources, it throws an error. This is due to it recursing into the sources list and becoming simple types with no fields. I added this as a sanity check to avoid that error if we happen to use attr.asdict for other purposes. If you don't think it's necessary, I'll remove it.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can leave it here and add a comment

# This can happen when attr.asdict recuses on basic types in list fields
# including the serialized result of the "sources" field
return value
if field.name == "sources":
return sorted(source.name for source in value)
if field.name == "target":
Expand Down
104 changes: 83 additions & 21 deletions test/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@
import os
import pytest
import responses
import attr

from cyberbrain import _TracerFSM, trace
from cyberbrain import _TracerFSM, trace, Symbol
from utils import python_version, get_os_type


Expand Down Expand Up @@ -40,37 +41,98 @@ def fixture_trace(request):
trace.tracer_state = _TracerFSM.INITIAL


def get_golden_data(golden_filepath, key):
directory = os.path.dirname(golden_filepath)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

type hints

if not os.path.exists(directory):
os.makedirs(directory)

if not os.path.exists(golden_filepath):
return None

with open(golden_filepath, "r") as f:
golden_frame_data = json.load(f)
return golden_frame_data.get(key)


def update_golden_data(golden_filepath, key, value):
golden_frame_data = {}
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

if os.path.exists(golden_filepath):
with open(golden_filepath, "r") as f:
golden_frame_data = json.load(f)

golden_frame_data[key] = value

with open(golden_filepath, "w") as f:
json.dump(golden_frame_data, f, ensure_ascii=False, indent=4)


def serialize_symbol(symbol):
snapshot = symbol.snapshot and {
"location": symbol.snapshot.location,
"events_pointer": symbol.snapshot.events_pointer,
}
return {"name": symbol.name, "snapshot": snapshot}


def get_serialized_events():
tracer_events = []
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this can be extracted to a separate function.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add a doc string with a simple example.

for event in trace.events:
event_dict = attr.asdict(event)
for key, val in event_dict.items():
if type(val) == Symbol:
event_dict[key] = serialize_symbol(val)
elif type(val) == list:
event_dict[key] = sorted(
[serialize_symbol(sym) for sym in val if type(sym) == Symbol],
key=lambda x: x["name"],
)
event_dict["__class__"] = event.__class__.__name__
tracer_events.append(event_dict)
return tracer_events


@pytest.fixture(scope="function")
def check_tracer_events():
yield

tracer_events = get_serialized_events()

golden_filepath = f"test/data/{python_version}/{trace.frame.frame_name}.json"
golden_tracer_events = get_golden_data(golden_filepath, "tracer.events")
if golden_tracer_events is None:
update_golden_data(golden_filepath, "tracer.events", tracer_events)
else:
assert tracer_events == golden_tracer_events, json.dumps(
tracer_events, indent=4
)


@pytest.fixture
def mocked_responses(request):
def check_response(request):
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel like this function is doing multiple things and we should break into smaller ones, something like

golden_response = get_golden_response()
if golden_response:
  assert ...
else:
  update_golden_response()

This makese the intent more clear. A good side effect: skip assertion if golden_response does not exist (now assertion still happens if the file exists).

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same with check_tracer_events

with responses.RequestsMock() as resp:
resp.add(
responses.POST,
f"http://localhost:{trace.rpc_client.port}/frame",
status=200,
content_type="application/octet-stream",
)
yield resp
yield

frame_data = msgpack.unpackb(resp.calls[0].request.body)
frame_name = frame_data["metadata"]["frame_name"]

# Generates golden data.
golden_filepath = f"test/data/{python_version}/{frame_name}.json"
directory = os.path.dirname(golden_filepath)
if not os.path.exists(directory):
os.makedirs(directory)

if not os.path.exists(golden_filepath):
with open(golden_filepath, "wt") as f:
json.dump(frame_data, f, ensure_ascii=False, indent=4)
return

# Assuming run in root directory.
with open(golden_filepath, "rt") as f:
golden_frame_data = json.loads(f.read())
response = msgpack.unpackb(resp.calls[0].request.body)
frame_name = response["metadata"]["frame_name"]

# Don't check request body on Windows because it has a different format.
if get_os_type() == "windows" and frame_name in {"test_numpy", "test_pandas"}:
return

assert frame_data == golden_frame_data, json.dumps(frame_data, indent=4)
golden_filepath = f"test/data/{python_version}/{frame_name}.json"
golden_response = get_golden_data(golden_filepath, "response")
if golden_response is None:
update_golden_data(golden_filepath, "response", response)
else:
assert response == golden_response, json.dumps(response, indent=4)


@pytest.fixture
def check_golden_file(check_tracer_events, check_response):
yield
66 changes: 66 additions & 0 deletions test/data/py37/attribute_error.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
{
"response": {
"metadata": {
"frame_id": "attribute_error",
"frame_name": "attribute_error",
"filename": "test_exception.py",
"defined_lineno": 72
},
"identifiers": [
"a"
],
"loops": [],
"events": [
{
"lineno": 74,
"index": 0,
"offset": 14,
"filename": "test_exception.py",
"id": "test_attribute_error:0",
"target": "a",
"value": "1",
"repr": "1",
"type": "Binding"
},
{
"lineno": 84,
"index": 1,
"offset": 126,
"filename": "test_exception.py",
"id": "test_attribute_error:1",
"value": "null",
"repr": "None",
"type": "Return"
}
],
"tracingResult": {}
},
"tracer.events": [
{
"lineno": 74,
"index": 0,
"offset": 14,
"filename": "test_exception.py",
"id": "test_attribute_error:0",
"target": {
"name": "a",
"snapshot": null
},
"value": "1",
"repr": "1",
"sources": [],
"__class__": "Binding"
},
{
"lineno": 84,
"index": 1,
"offset": 126,
"filename": "test_exception.py",
"id": "test_attribute_error:1",
"value": "null",
"repr": "None",
"sources": [],
"__class__": "Return"
}
]
}
38 changes: 38 additions & 0 deletions test/data/py37/binary_op_zero_division.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"response": {
"metadata": {
"frame_id": "binary_op_zero_division",
"frame_name": "binary_op_zero_division",
"filename": "test_exception.py",
"defined_lineno": 19
},
"identifiers": [],
"loops": [],
"events": [
{
"lineno": 21,
"index": 0,
"offset": 32,
"filename": "test_exception.py",
"id": "test_binary_op_zero_division:0",
"value": "null",
"repr": "None",
"type": "Return"
}
],
"tracingResult": {}
},
"tracer.events": [
{
"lineno": 21,
"index": 0,
"offset": 32,
"filename": "test_exception.py",
"id": "test_binary_op_zero_division:0",
"value": "null",
"repr": "None",
"sources": [],
"__class__": "Return"
}
]
}
38 changes: 38 additions & 0 deletions test/data/py37/build_map_type_error.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"response": {
"metadata": {
"frame_id": "build_map_type_error",
"frame_name": "build_map_type_error",
"filename": "test_exception.py",
"defined_lineno": 100
},
"identifiers": [],
"loops": [],
"events": [
{
"lineno": 102,
"index": 0,
"offset": 34,
"filename": "test_exception.py",
"id": "test_build_map_type_error:0",
"value": "null",
"repr": "None",
"type": "Return"
}
],
"tracingResult": {}
},
"tracer.events": [
{
"lineno": 102,
"index": 0,
"offset": 34,
"filename": "test_exception.py",
"id": "test_build_map_type_error:0",
"value": "null",
"repr": "None",
"sources": [],
"__class__": "Return"
}
]
}
38 changes: 38 additions & 0 deletions test/data/py37/build_set_type_error.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"response": {
"metadata": {
"frame_id": "build_set_type_error",
"frame_name": "build_set_type_error",
"filename": "test_exception.py",
"defined_lineno": 91
},
"identifiers": [],
"loops": [],
"events": [
{
"lineno": 93,
"index": 0,
"offset": 30,
"filename": "test_exception.py",
"id": "test_build_set_type_error:0",
"value": "null",
"repr": "None",
"type": "Return"
}
],
"tracingResult": {}
},
"tracer.events": [
{
"lineno": 93,
"index": 0,
"offset": 30,
"filename": "test_exception.py",
"id": "test_build_set_type_error:0",
"value": "null",
"repr": "None",
"sources": [],
"__class__": "Return"
}
]
}
66 changes: 66 additions & 0 deletions test/data/py37/call_method_type_error.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
{
"response": {
"metadata": {
"frame_id": "call_method_type_error",
"frame_name": "call_method_type_error",
"filename": "test_exception.py",
"defined_lineno": 8
},
"identifiers": [
"s"
],
"loops": [],
"events": [
{
"lineno": 9,
"index": 0,
"offset": 2,
"filename": "test_exception.py",
"id": "test_call_method_type_error:0",
"target": "s",
"value": "\"hello world\"",
"repr": "\"hello world\"",
"type": "Binding"
},
{
"lineno": 12,
"index": 1,
"offset": 58,
"filename": "test_exception.py",
"id": "test_call_method_type_error:1",
"value": "null",
"repr": "None",
"type": "Return"
}
],
"tracingResult": {}
},
"tracer.events": [
{
"lineno": 9,
"index": 0,
"offset": 2,
"filename": "test_exception.py",
"id": "test_call_method_type_error:0",
"target": {
"name": "s",
"snapshot": null
},
"value": "\"hello world\"",
"repr": "\"hello world\"",
"sources": [],
"__class__": "Binding"
},
{
"lineno": 12,
"index": 1,
"offset": 58,
"filename": "test_exception.py",
"id": "test_call_method_type_error:1",
"value": "null",
"repr": "None",
"sources": [],
"__class__": "Return"
}
]
}
Loading