From 0293c4acc236ec3fc3bc1dac1b4e9e34e33cd97d Mon Sep 17 00:00:00 2001 From: Miguel Date: Fri, 14 Feb 2025 16:33:43 +0100 Subject: [PATCH 1/6] feat: :sparkles: added graphql logic for abfall.io changed current vars to graphql variables see #3735 --- .../source/abfall_io_graphql.py | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py new file mode 100644 index 000000000..99968c859 --- /dev/null +++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py @@ -0,0 +1,132 @@ +import datetime +import logging +import re +from html.parser import HTMLParser + +import requests +from waste_collection_schedule import Collection # type: ignore[attr-defined] +from waste_collection_schedule.service.AbfallIO import SERVICE_MAP +from waste_collection_schedule.service.ICS import ICS + +TITLE = "Abfall.IO GraphQL" +DESCRIPTION = ( + "Source for AbfallPlus.de waste collection. Service is hosted on abfall.io." +) +URL = "https://www.abfallplus.de" +COUNTRY = "de" + + +def EXTRA_INFO(): + return [ + { + "title": s["title"], + "url": s["url"], + "default_params": {"key": s["service_id"]}, + } + for s in SERVICE_MAP + ] + + +TEST_CASES = { + "Neckarsulm": { + "key": "18adb00cb5135f6aa16b5fdea6dae2c63a507ae0f836540e", + "idHouseNumber": 304, + # "wasteTypes": ["28", "19", "31", "654"] + }, + "Weinsberg": { + "key": "18adb00cb5135f6aa16b5fdea6dae2c63a507ae0f836540e", + "idHouseNumber": 353, + # "wasteTypes": ["28", "19", "31", "654"] + }, +} +_LOGGER = logging.getLogger(__name__) + +MODUS_KEY = "d6c5855a62cf32a4dadbc2831f0f295f" +HEADERS = {"user-agent": "Mozilla/5.0 (xxxx Windows NT 10.0; Win64; x64)"} + + +# Parser for HTML input (hidden) text +class HiddenInputParser(HTMLParser): + def __init__(self): + super().__init__() + self._args = {} + + @property + def args(self): + return self._args + + def handle_starttag(self, tag, attrs): + if tag == "input": + d = dict(attrs) + if d["type"] == "hidden": + self._args[d["name"]] = d["value"] + + +class Source: + def __init__( + self, + key, + idHouseNumber, + wasteTypes=[], + ): + self._key = key + self._idHouseNumber = idHouseNumber + self._wasteTypes = wasteTypes # list of integers + self._ics = ICS() + + def fetch(self): + # get token + params = {"key": self._key, "modus": MODUS_KEY, "waction": "init"} + + r = requests.post("https://widgets.abfall.io/graphql", + params=params, headers=HEADERS) + + # add all hidden input fields to form data + # There is one hidden field which acts as a token: + # It consists of a UUID key and a UUID value. + p = HiddenInputParser() + p.feed(r.text) + args = p.args + + args["idHouseNumber"] = self._idHouseNumber + + for i in range(len(self._wasteTypes)): + args[f"f_id_abfalltyp_{i}"] = self._wasteTypes[i] + + args["wasteTypes_index_max"] = len(self._wasteTypes) + args["wasteTypes"] = ",".join( + map(lambda x: str(x), self._wasteTypes)) + + now = datetime.datetime.now() + date2 = now + datetime.timedelta(days=365) + args["timeperiod"] = f"{now.strftime('%Y%m%d')}-{date2.strftime('%Y%m%d')}" + + params = {"key": self._key, "modus": MODUS_KEY, + "waction": "export_ics"} + + # get csv file + r = requests.post( + "https://widgets.abfall.io/graphql", params=params, data=args, headers=HEADERS + ) + + # parse ics file + r.encoding = "utf-8" # requests doesn't guess the encoding correctly + ics_file = r.text + + # Remove all lines starting with " + # waste type. The warning could be removed by adding the extra config + # option "f_abfallarten" with the following values [27, 28, 17, 67] + html_warnings = re.findall(r"\ Date: Fri, 14 Feb 2025 16:36:36 +0100 Subject: [PATCH 2/6] refactor: :recycle: changed landkreis hn to graphql service --- .../waste_collection_schedule/service/AbfallIO.py | 5 ----- .../service/AbfallIOGraphQL.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 custom_components/waste_collection_schedule/waste_collection_schedule/service/AbfallIOGraphQL.py diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/service/AbfallIO.py b/custom_components/waste_collection_schedule/waste_collection_schedule/service/AbfallIO.py index 19e152408..73e3535e9 100644 --- a/custom_components/waste_collection_schedule/waste_collection_schedule/service/AbfallIO.py +++ b/custom_components/waste_collection_schedule/waste_collection_schedule/service/AbfallIO.py @@ -42,11 +42,6 @@ "url": "https://www.geb-goettingen.de/", "service_id": "7dd0d724cbbd008f597d18fcb1f474cb", }, - { - "title": "Landkreis Heilbronn", - "url": "https://www.landkreis-heilbronn.de/", - "service_id": "1a1e7b200165683738adddc4bd0199a2", - }, { "title": "Abfallwirtschaft Landkreis Kitzingen", "url": "https://www.abfallwelt.de/", diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/service/AbfallIOGraphQL.py b/custom_components/waste_collection_schedule/waste_collection_schedule/service/AbfallIOGraphQL.py new file mode 100644 index 000000000..4f81e69d1 --- /dev/null +++ b/custom_components/waste_collection_schedule/waste_collection_schedule/service/AbfallIOGraphQL.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + + +SERVICE_MAP = [ + { + "title": "Landkreis Heilbronn", + "url": "https://www.landkreis-heilbronn.de/", + "service_id": "1a1e7b200165683738adddc4bd0199a2", + }, +] From f6fa2b3031c04a476a9ecf19199532cb451ddf07 Mon Sep 17 00:00:00 2001 From: Miguel Date: Fri, 14 Feb 2025 16:37:08 +0100 Subject: [PATCH 3/6] feat: :sparkles: changed service map to graphql --- .../waste_collection_schedule/source/abfall_io_graphql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py index 99968c859..017d1fa77 100644 --- a/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py +++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py @@ -5,7 +5,7 @@ import requests from waste_collection_schedule import Collection # type: ignore[attr-defined] -from waste_collection_schedule.service.AbfallIO import SERVICE_MAP +from waste_collection_schedule.service.AbfallIOGraphQL import SERVICE_MAP from waste_collection_schedule.service.ICS import ICS TITLE = "Abfall.IO GraphQL" From 900d4b8d78ebcd3ca732dc078fb411c486cb0a63 Mon Sep 17 00:00:00 2001 From: Miguel Date: Fri, 14 Feb 2025 16:38:08 +0100 Subject: [PATCH 4/6] refactor: :recycle: changed api url 4 csv file --- .../waste_collection_schedule/source/abfall_io_graphql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py index 017d1fa77..f822e3a92 100644 --- a/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py +++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py @@ -106,7 +106,7 @@ def fetch(self): # get csv file r = requests.post( - "https://widgets.abfall.io/graphql", params=params, data=args, headers=HEADERS + "https://api.abfall.io", params=params, data=args, headers=HEADERS ) # parse ics file From d612ff474fe69f281c3318c17e26602ce5810dcf Mon Sep 17 00:00:00 2001 From: Miguel Date: Fri, 14 Feb 2025 16:43:06 +0100 Subject: [PATCH 5/6] docs: :memo: update documentation --- doc/source/abfall_io_graphql.md | 67 +++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 doc/source/abfall_io_graphql.md diff --git a/doc/source/abfall_io_graphql.md b/doc/source/abfall_io_graphql.md new file mode 100644 index 000000000..fdbdf5aa2 --- /dev/null +++ b/doc/source/abfall_io_graphql.md @@ -0,0 +1,67 @@ +# Abfall.IO /AbfallPlus - GraphQL + +Support for schedules provided by [Abfall.IO](https://abfall.io). The official homepage is using the URL [AbfallPlus.de](https://www.abfallplus.de/) instead. + +This source is designed for the old API of Abfall.IO. If your region/preovider uses the new API you should use the [Abfall ICS version](/doc/ics/abfall_io_ics.md) source instead. Your provider uses the new API if you can see an `ICS` button above your collection dates on the website after selecting your location and waste types. + +## Configuration via configuration.yaml + +```yaml +waste_collection_schedule: + sources: + - name: abfall_io + args: + key: KEY + idHouseNumber: HOUSE_NUMBER_ID + wasteTypes: + - 1 + - 2 + - 3 +``` + +### Configuration Variables + +**key** +*(hash) (required)* + +**idHouseNumber** +*(integer) (required)* + +**wasteTypes** +*(list of integer) (optional)* + +## Example + +```yaml +waste_collection_schedule: + sources: + - name: abfall_io + args: + key: "8215c62763967916979e0e8566b6172e" + idHouseNumber: 304 +``` + +## How to get the source arguments + +## Simple Variant: Use wizard script + +There is a script with an interactive command line interface which generates the required source configuration: + +[https://github.com/mampfes/hacs_waste_collection_schedule/blob/master/custom_components/waste_collection_schedule/waste_collection_schedule/wizard/abfall_io_graphql.py](https://github.com/mampfes/hacs_waste_collection_schedule/blob/master/custom_components/waste_collection_schedule/waste_collection_schedule/wizard/abfall_io_graphql.py). + +First, install the Python module `inquirer`. Then run this script from a shell and answer the questions. + +### Hardcore Variant: Extract arguments from website + +Another way get the source arguments is to us a (desktop) browser with developer tools, e.g. Google Chrome: + +1. Open your county's `Abfuhrtermine` homepage, e.g. [https://www.lrabb.de/start/Service+_+Verwaltung/Abfuhrtermine.html](https://www.lrabb.de/start/Service+_+Verwaltung/Abfuhrtermine.html). +2. Enter your data, but don't click on `Datei exportieren` so far! +3. Select `Exportieren als`: `ICS` +4. Open the Developer Tools (Ctrl + Shift + I) and open the `Network` tab. +5. Now click the `Datei exportieren` button. +6. You should see one entry in the network recording. +7. Select the entry on the left hand side and scroll down to `Query String Parameters` on the right hand side. +8. Here you can find the value for `key`. +9. Now go down to the next section `Form Data`. +10. Here you can find the values for `idHouseNumber`, `wasteTypes`. All other entries don't care. From f892a890a29a3704dca48d34cccfa3f4d96cda74 Mon Sep 17 00:00:00 2001 From: Miguel Date: Fri, 14 Feb 2025 16:48:24 +0100 Subject: [PATCH 6/6] feat: :sparkles: added body to request --- .../source/abfall_io_graphql.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py index f822e3a92..203f2eefc 100644 --- a/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py +++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/abfall_io_graphql.py @@ -78,8 +78,22 @@ def fetch(self): # get token params = {"key": self._key, "modus": MODUS_KEY, "waction": "init"} + now = datetime.datetime.now() + date2 = now + datetime.timedelta(days=365) + args["timeperiod"] = f"{now.strftime('%Y%m%d')}-{date2.strftime('%Y%m%d')}" + + body = { + "query": "\nquery Query($idHouseNumber: ID!, $wasteTypes: [ID], $dateMin: Date, $dateMax: Date, $showInactive: Boolean) {\nappointments(idHouseNumber: $idHouseNumber, wasteTypes: $wasteTypes, dateMin: $dateMin, dateMax: $dateMax, showInactive: $showInactive) {\nid\ndate\ntime\nlocation\nnote\nwasteType {\nid\nname\ncolor\ninternals {\npdfLegend\niconLow\n}\n}\n}\n}\n", + "variables": { + "idHouseNumber": self._idHouseNumber, + "wasteTypes": self._wasteTypes, + "dateMin": now.strftime("%Y-%m-%d"), + "dateMax": date2.strftime("%Y-%m-%d"), + } + } + r = requests.post("https://widgets.abfall.io/graphql", - params=params, headers=HEADERS) + params=params, headers=HEADERS, data=body) # add all hidden input fields to form data # There is one hidden field which acts as a token: @@ -97,10 +111,6 @@ def fetch(self): args["wasteTypes"] = ",".join( map(lambda x: str(x), self._wasteTypes)) - now = datetime.datetime.now() - date2 = now + datetime.timedelta(days=365) - args["timeperiod"] = f"{now.strftime('%Y%m%d')}-{date2.strftime('%Y%m%d')}" - params = {"key": self._key, "modus": MODUS_KEY, "waction": "export_ics"}