From 2e3e5298410bd2937f44453a2b9a80521bceb0c5 Mon Sep 17 00:00:00 2001 From: BigOrangeQWQ <2284086963@qq.com> Date: Fri, 15 Nov 2024 20:38:26 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=B0=86=20load=5Fjson=20=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E7=BB=9F=E4=B8=80=E4=B8=BA=20json5=20(#262)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: 将 load_json 函数统一为 json5 * style: auto fix by pre-commit hooks * fix: 修复部分函数未适配至 load_json5 * fix: 修复 import 路径问题 * fix: 将网络请求 json 的解析统一为 json5 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- src/plugins/github/plugins/publish/utils.py | 4 +-- .../github/plugins/publish/validation.py | 8 +++--- .../github/plugins/remove/validation.py | 8 +++--- src/providers/docker_test/__init__.py | 6 +++-- src/providers/store_test/store.py | 10 +++---- src/providers/store_test/utils.py | 4 +-- src/providers/utils.py | 27 +++++++------------ src/providers/validation/models.py | 12 +++++---- src/providers/validation/utils.py | 11 ++++---- tests/utils/store_test/test_utils.py | 4 +-- 10 files changed, 44 insertions(+), 50 deletions(-) diff --git a/src/plugins/github/plugins/publish/utils.py b/src/plugins/github/plugins/publish/utils.py index ce8748a5..55f52170 100644 --- a/src/plugins/github/plugins/publish/utils.py +++ b/src/plugins/github/plugins/publish/utils.py @@ -12,7 +12,7 @@ from src.plugins.github.utils import commit_message as _commit_message from src.plugins.github.utils import run_shell_command from src.providers.models import RegistryUpdatePayload, to_store -from src.providers.utils import dump_json5, load_json5_from_file +from src.providers.utils import dump_json5, load_json_from_file from src.providers.validation import PublishType, ValidationDict from .constants import ( @@ -150,7 +150,7 @@ def update_file(result: ValidationDict) -> None: logger.info(f"正在更新文件: {path}") - data = load_json5_from_file(path) + data = load_json_from_file(path) data.append(new_data) dump_json5(path, data) diff --git a/src/plugins/github/plugins/publish/validation.py b/src/plugins/github/plugins/publish/validation.py index 4fa1b0eb..f5280beb 100644 --- a/src/plugins/github/plugins/publish/validation.py +++ b/src/plugins/github/plugins/publish/validation.py @@ -9,7 +9,7 @@ from src.plugins.github.models.issue import IssueHandler from src.plugins.github.utils import extract_issue_info_from_issue from src.providers.docker_test import DockerPluginTest, Metadata -from src.providers.utils import load_json5_from_file +from src.providers.utils import load_json_from_file from src.providers.validation import PublishType, ValidationDict, validate_info from .constants import ( @@ -64,7 +64,7 @@ async def validate_plugin_info_from_issue( test_config: str = raw_data.get("test_config", "") # 获取插件上次的数据 - previous_data = load_json5_from_file(plugin_config.input_config.plugin_path) + previous_data = load_json_from_file(plugin_config.input_config.plugin_path) # 决定是否跳过插件测试 # 因为在上一步可能已经知道了是否跳过插件测试，所以这里可以传入 @@ -146,7 +146,7 @@ async def validate_adapter_info_from_issue(issue: Issue) -> ValidationDict: ) raw_data.update(AuthorInfo.from_issue(issue).model_dump()) - previous_data = load_json5_from_file(plugin_config.input_config.adapter_path) + previous_data = load_json_from_file(plugin_config.input_config.adapter_path) return validate_info(PublishType.ADAPTER, raw_data, previous_data) @@ -166,6 +166,6 @@ async def validate_bot_info_from_issue(issue: Issue) -> ValidationDict: raw_data.update(AuthorInfo.from_issue(issue).model_dump()) - previous_data = load_json5_from_file(plugin_config.input_config.bot_path) + previous_data = load_json_from_file(plugin_config.input_config.bot_path) return validate_info(PublishType.BOT, raw_data, previous_data) diff --git a/src/plugins/github/plugins/remove/validation.py b/src/plugins/github/plugins/remove/validation.py index 59fa257a..aa582d93 100644 --- a/src/plugins/github/plugins/remove/validation.py +++ b/src/plugins/github/plugins/remove/validation.py @@ -6,7 +6,7 @@ from src.plugins.github.models import AuthorInfo from src.plugins.github.utils import extract_issue_info_from_issue from src.providers.constants import BOT_KEY_TEMPLATE, PYPI_KEY_TEMPLATE -from src.providers.utils import load_json5_from_file +from src.providers.utils import load_json_from_file from src.providers.validation.models import PublishType from .constants import ( @@ -26,7 +26,7 @@ def load_publish_data(publish_type: PublishType): project_link=adapter["project_link"], module_name=adapter["module_name"], ): adapter - for adapter in load_json5_from_file( + for adapter in load_json_from_file( plugin_config.input_config.adapter_path ) } @@ -36,7 +36,7 @@ def load_publish_data(publish_type: PublishType): name=bot["name"], homepage=bot["homepage"], ): bot - for bot in load_json5_from_file(plugin_config.input_config.bot_path) + for bot in load_json_from_file(plugin_config.input_config.bot_path) } case PublishType.PLUGIN: return { @@ -44,7 +44,7 @@ def load_publish_data(publish_type: PublishType): project_link=plugin["project_link"], module_name=plugin["module_name"], ): plugin - for plugin in load_json5_from_file( + for plugin in load_json_from_file( plugin_config.input_config.plugin_path ) } diff --git a/src/providers/docker_test/__init__.py b/src/providers/docker_test/__init__.py index 690d7f18..264f2ab1 100644 --- a/src/providers/docker_test/__init__.py +++ b/src/providers/docker_test/__init__.py @@ -4,8 +4,10 @@ import docker from pydantic import BaseModel, Field, field_validator, model_validator from pydantic_core import PydanticCustomError +from pyjson5 import Json5DecoderException from src.providers.constants import DOCKER_IMAGES, REGISTRY_PLUGINS_URL +from src.providers.utils import load_json class Metadata(BaseModel): @@ -29,8 +31,8 @@ def model_validator(cls, data: dict[str, Any]): def supported_adapters_validator(cls, v: list[str] | str | None): if isinstance(v, str): try: - v = json.loads(v) - except json.JSONDecodeError: + v = load_json(v) + except Json5DecoderException: raise PydanticCustomError("json_type", "JSON 格式不合法") return v diff --git a/src/providers/store_test/store.py b/src/providers/store_test/store.py index fce47b17..efb237b3 100644 --- a/src/providers/store_test/store.py +++ b/src/providers/store_test/store.py @@ -26,7 +26,7 @@ StorePlugin, StoreTestResult, ) -from src.providers.utils import dump_json, load_json5_from_web, load_json_from_web +from src.providers.utils import dump_json, load_json_from_web from src.providers.validation.utils import get_author_name from .constants import ( @@ -53,28 +53,28 @@ def __init__(self) -> None: project_link=adapter["project_link"], module_name=adapter["module_name"], ): StoreAdapter(**adapter) - for adapter in load_json5_from_web(STORE_ADAPTERS_URL) + for adapter in load_json_from_web(STORE_ADAPTERS_URL) } self._store_bots: dict[str, StoreBot] = { BOT_KEY_TEMPLATE.format( name=bot["name"], homepage=bot["homepage"], ): StoreBot(**bot) - for bot in load_json5_from_web(STORE_BOTS_URL) + for bot in load_json_from_web(STORE_BOTS_URL) } self._store_drivers: dict[str, StoreDriver] = { PYPI_KEY_TEMPLATE.format( project_link=driver["project_link"], module_name=driver["module_name"], ): StoreDriver(**driver) - for driver in load_json5_from_web(STORE_DRIVERS_URL) + for driver in load_json_from_web(STORE_DRIVERS_URL) } self._store_plugins: dict[str, StorePlugin] = { PYPI_KEY_TEMPLATE.format( project_link=plugin["project_link"], module_name=plugin["module_name"], ): StorePlugin(**plugin) - for plugin in load_json5_from_web(STORE_PLUGINS_URL) + for plugin in load_json_from_web(STORE_PLUGINS_URL) } # 上次测试的结果 self._previous_results: dict[str, StoreTestResult] = { diff --git a/src/providers/store_test/utils.py b/src/providers/store_test/utils.py index 0602cacc..49ee322d 100644 --- a/src/providers/store_test/utils.py +++ b/src/providers/store_test/utils.py @@ -3,7 +3,7 @@ import httpx -from src.providers.utils import load_json_from_web +from src.providers.utils import load_json, load_json_from_web @cache @@ -16,7 +16,7 @@ def get_pypi_data(project_link: str) -> dict[str, Any]: r = httpx.get(url, headers=headers) if r.status_code != 200: raise ValueError(f"获取 PyPI 数据失败：{r.text}") - return r.json() + return load_json(r.text) def get_latest_version(project_link: str) -> str: diff --git a/src/providers/utils.py b/src/providers/utils.py index aee93d07..7b75d12f 100644 --- a/src/providers/utils.py +++ b/src/providers/utils.py @@ -8,17 +8,22 @@ def load_json_from_file(file_path: Path): - """从文件加载 JSON 文件""" + """从文件加载 JSON5 文件""" with open(file_path, encoding="utf-8") as file: - return json.load(file) + return pyjson5.decode_io(file) # type: ignore def load_json_from_web(url: str): - """从网络加载 JSON 文件""" + """从网络加载 JSON5 文件""" r = httpx.get(url) if r.status_code != 200: raise ValueError(f"下载文件失败：{r.text}") - return r.json() + return pyjson5.decode(r.text) + + +def load_json(text: str): + """从文本加载 JSON5""" + return pyjson5.decode(text) def dump_json(path: Path, data: Any, minify: bool = True) -> None: @@ -34,20 +39,6 @@ def dump_json(path: Path, data: Any, minify: bool = True) -> None: json.dump(data, f, ensure_ascii=False, indent=2) -def load_json5_from_file(file_path: Path): - """从文件加载 JSON5 文件""" - with open(file_path, encoding="utf-8") as file: - return pyjson5.decode_io(file) # type: ignore - - -def load_json5_from_web(url: str): - """从网络加载 JSON5 文件""" - r = httpx.get(url) - if r.status_code != 200: - raise ValueError(f"下载文件失败：{r.text}") - return pyjson5.decode(r.text) - - def dump_json5(path: Path, data: Any) -> None: """保存 JSON5 文件 diff --git a/src/providers/validation/models.py b/src/providers/validation/models.py index 7eaffc3f..fdb1562d 100644 --- a/src/providers/validation/models.py +++ b/src/providers/validation/models.py @@ -1,6 +1,5 @@ # ruff: noqa: UP040 import abc -import json from enum import Enum from typing import Annotated, Any, TypeAlias @@ -17,6 +16,9 @@ ) from pydantic_core import ErrorDetails, PydanticCustomError, to_jsonable_python from pydantic_extra_types.color import Color +from pyjson5 import Json5DecoderException + +from src.providers.utils import load_json from .constants import ( NAME_MAX_LENGTH, @@ -188,8 +190,8 @@ def tags_validator(cls, v: str | list[Any]) -> list[dict[str, str]]: return v try: - return json.loads(v) - except json.JSONDecodeError: + return load_json(v) + except Json5DecoderException: raise PydanticCustomError("json_type", "JSON 格式不合法") @@ -236,8 +238,8 @@ def supported_adapters_validator( # 如果是从 issue 中获取的数据，需要先解码 if skip_test and isinstance(v, str): try: - v = json.loads(v) - except json.JSONDecodeError: + v = load_json(v) + except Json5DecoderException: raise PydanticCustomError("json_type", "JSON 格式不合法") # 如果是支持所有适配器，值应该是 None，不需要检查 diff --git a/src/providers/validation/utils.py b/src/providers/validation/utils.py index a25a7864..723c889f 100644 --- a/src/providers/validation/utils.py +++ b/src/providers/validation/utils.py @@ -4,6 +4,7 @@ import httpx from src.providers.constants import STORE_ADAPTERS_URL +from src.providers.utils import load_json, load_json_from_web from .constants import MESSAGE_TRANSLATIONS @@ -22,7 +23,7 @@ def get_pypi_name(project_link: str) -> str: url = f"https://pypi.org/pypi/{project_link}/json" r = get_url(url) r.raise_for_status() - data = r.json() + data = load_json(r.text) return data["info"]["name"] @@ -33,7 +34,7 @@ def get_upload_time(project_link: str) -> str | None: if r.status_code != 200: return None try: - data = r.json() + data = load_json(r.text) except Exception: return None return data["urls"][0]["upload_time_iso_8601"] @@ -62,14 +63,12 @@ def check_url(url: str) -> tuple[int, str]: def get_author_name(author_id: int) -> str: """通过作者的ID获取作者名字""" url = f"https://api.github.com/user/{author_id}" - resp = httpx.get(url) - return resp.json()["login"] + return load_json_from_web(url)["login"] def get_adapters() -> set[str]: """获取适配器列表""" - resp = httpx.get(STORE_ADAPTERS_URL) - adapters = resp.json() + adapters = load_json_from_web(STORE_ADAPTERS_URL) return {adapter["module_name"] for adapter in adapters} diff --git a/tests/utils/store_test/test_utils.py b/tests/utils/store_test/test_utils.py index fa634f7c..5e74706c 100644 --- a/tests/utils/store_test/test_utils.py +++ b/tests/utils/store_test/test_utils.py @@ -6,12 +6,12 @@ async def test_load_json_failed(mocked_api: MockRouter): """测试加载 json 失败""" - from src.providers.utils import load_json5_from_web + from src.providers.utils import load_json_from_web mocked_api.get(STORE_ADAPTERS_URL).respond(404) with pytest.raises(ValueError, match="下载文件失败："): - load_json5_from_web(STORE_ADAPTERS_URL) + load_json_from_web(STORE_ADAPTERS_URL) async def test_get_pypi_data_failed(mocked_api: MockRouter):