-
Notifications
You must be signed in to change notification settings - Fork 103
/
Copy pathtest.py
117 lines (106 loc) · 3.9 KB
/
test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
from cerberus import Validator
import immowelt
import pytest
import pprint
pp = pprint.PrettyPrinter(indent=4)
def validate_or_fail(item, validator):
if not validator.validate(item):
pp.pformat(item)
pytest.fail(f"Validation failed for item: {pp.pformat(item)}\nErrors: {validator.errors}")
proeprty_schema = {
"schema": {
"type": "dict",
"schema": {
"brand": {"type": "string"},
"id": {"type": "string"},
"title": {"type": "string"},
"sections": {
"type": "dict",
"schema": {
"location": {
"type": "dict",
"schema": {
"address": {
"type": "dict",
"schema": {
"country": {"type": "string"},
"city": {"type": "string"},
"zipCode": {"type": "string"},
"street": {"type": "string"},
"district": {"type": "string"},
}
}
}
},
"features": {
"type": "dict",
"schema": {
"preview": {
"type": "list",
"schema": {
"type": "dict",
"schema": {
"icon": {"type": "string"},
"value": {"type": "string"},
}
}
}
}
}
}
}
}
}
}
search_schema = {
"schema": {
"type": "dict",
"schema": {
"brand": {"type": "string"},
"id": {"type": "string"},
"status": {"type": "string"},
"location": {
"type": "dict",
"schema": {
"address": {
"type": "dict",
"schema": {
"country": {"type": "string"},
"city": {"type": "string"},
"zipCode": {"type": "string"},
"street": {"type": "string"},
"district": {"type": "string"},
}
}
}
},
},
}
}
@pytest.mark.asyncio
@pytest.mark.flaky(reruns=3, reruns_delay=30)
async def test_properties_scraping():
properties_data = await immowelt.scrape_properties(
urls=[
"https://www.immowelt.de/expose/27t9c5f",
"https://www.immowelt.de/expose/27dgc5f",
"https://www.immowelt.de/expose/9175275c-9b96-454f-a770-7f4ef0e720be",
"https://www.immowelt.de/expose/95aba3fb-8449-47d3-8394-9ab71e705160",
"https://www.immowelt.de/expose/ac9dc8d0-a729-4d79-849e-93ec9d4cf16a"
]
)
validator = Validator(proeprty_schema, allow_unknown=True)
for item in properties_data:
validate_or_fail(item, validator)
assert len(properties_data) >= 1
@pytest.mark.asyncio
@pytest.mark.flaky(reruns=3, reruns_delay=30)
async def test_search_scraping():
search_data = await immowelt.scrape_search(
url="https://www.immowelt.de/classified-search?distributionTypes=Buy&estateTypes=Apartment&locations=AD08DE6345",
max_scrape_pages=3
)
validator = Validator(search_schema, allow_unknown=True)
for item in search_data:
validate_or_fail(item, validator)
assert len(search_data) >= 2