Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: 将 load_json 函数统一为 json5 #262

Merged
merged 5 commits into from
Nov 15, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/plugins/github/plugins/publish/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from src.plugins.github.utils import commit_message as _commit_message
from src.plugins.github.utils import run_shell_command
from src.providers.models import RegistryUpdatePayload, to_store
from src.providers.utils import dump_json5, load_json5_from_file
from src.providers.utils import dump_json5, load_json_from_file
from src.providers.validation import PublishType, ValidationDict

from .constants import (
Expand Down Expand Up @@ -150,7 +150,7 @@ def update_file(result: ValidationDict) -> None:

logger.info(f"正在更新文件: {path}")

data = load_json5_from_file(path)
data = load_json_from_file(path)
data.append(new_data)
dump_json5(path, data)

Expand Down
8 changes: 4 additions & 4 deletions src/plugins/github/plugins/publish/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from src.plugins.github.models.issue import IssueHandler
from src.plugins.github.utils import extract_issue_info_from_issue
from src.providers.docker_test import DockerPluginTest, Metadata
from src.providers.utils import load_json5_from_file
from src.providers.utils import load_json_from_file
from src.providers.validation import PublishType, ValidationDict, validate_info

from .constants import (
Expand Down Expand Up @@ -64,7 +64,7 @@ async def validate_plugin_info_from_issue(
test_config: str = raw_data.get("test_config", "")

# 获取插件上次的数据
previous_data = load_json5_from_file(plugin_config.input_config.plugin_path)
previous_data = load_json_from_file(plugin_config.input_config.plugin_path)

# 决定是否跳过插件测试
# 因为在上一步可能已经知道了是否跳过插件测试,所以这里可以传入
Expand Down Expand Up @@ -146,7 +146,7 @@ async def validate_adapter_info_from_issue(issue: Issue) -> ValidationDict:
)
raw_data.update(AuthorInfo.from_issue(issue).model_dump())

previous_data = load_json5_from_file(plugin_config.input_config.adapter_path)
previous_data = load_json_from_file(plugin_config.input_config.adapter_path)

return validate_info(PublishType.ADAPTER, raw_data, previous_data)

Expand All @@ -166,6 +166,6 @@ async def validate_bot_info_from_issue(issue: Issue) -> ValidationDict:

raw_data.update(AuthorInfo.from_issue(issue).model_dump())

previous_data = load_json5_from_file(plugin_config.input_config.bot_path)
previous_data = load_json_from_file(plugin_config.input_config.bot_path)

return validate_info(PublishType.BOT, raw_data, previous_data)
8 changes: 4 additions & 4 deletions src/plugins/github/plugins/remove/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from src.plugins.github.models import AuthorInfo
from src.plugins.github.utils import extract_issue_info_from_issue
from src.providers.constants import BOT_KEY_TEMPLATE, PYPI_KEY_TEMPLATE
from src.providers.utils import load_json5_from_file
from src.providers.utils import load_json_from_file
from src.providers.validation.models import PublishType

from .constants import (
Expand All @@ -26,7 +26,7 @@ def load_publish_data(publish_type: PublishType):
project_link=adapter["project_link"],
module_name=adapter["module_name"],
): adapter
for adapter in load_json5_from_file(
for adapter in load_json_from_file(
plugin_config.input_config.adapter_path
)
}
Expand All @@ -36,15 +36,15 @@ def load_publish_data(publish_type: PublishType):
name=bot["name"],
homepage=bot["homepage"],
): bot
for bot in load_json5_from_file(plugin_config.input_config.bot_path)
for bot in load_json_from_file(plugin_config.input_config.bot_path)
}
case PublishType.PLUGIN:
return {
PYPI_KEY_TEMPLATE.format(
project_link=plugin["project_link"],
module_name=plugin["module_name"],
): plugin
for plugin in load_json5_from_file(
for plugin in load_json_from_file(
plugin_config.input_config.plugin_path
)
}
Expand Down
6 changes: 4 additions & 2 deletions src/providers/docker_test/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
import docker
from pydantic import BaseModel, Field, field_validator, model_validator
from pydantic_core import PydanticCustomError
from pyjson5 import Json5DecoderException

from src.providers.constants import DOCKER_IMAGES, REGISTRY_PLUGINS_URL
from src.providers.utils import load_json


class Metadata(BaseModel):
Expand All @@ -29,8 +31,8 @@
def supported_adapters_validator(cls, v: list[str] | str | None):
if isinstance(v, str):
try:
v = json.loads(v)
except json.JSONDecodeError:
v = load_json(v)
except Json5DecoderException:

Check warning on line 35 in src/providers/docker_test/__init__.py

View check run for this annotation

Codecov / codecov/patch

src/providers/docker_test/__init__.py#L34-L35

Added lines #L34 - L35 were not covered by tests
raise PydanticCustomError("json_type", "JSON 格式不合法")

return v
Expand Down
10 changes: 5 additions & 5 deletions src/providers/store_test/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
StorePlugin,
StoreTestResult,
)
from src.providers.utils import dump_json, load_json5_from_web, load_json_from_web
from src.providers.utils import dump_json, load_json_from_web
from src.providers.validation.utils import get_author_name

from .constants import (
Expand All @@ -53,28 +53,28 @@ def __init__(self) -> None:
project_link=adapter["project_link"],
module_name=adapter["module_name"],
): StoreAdapter(**adapter)
for adapter in load_json5_from_web(STORE_ADAPTERS_URL)
for adapter in load_json_from_web(STORE_ADAPTERS_URL)
}
self._store_bots: dict[str, StoreBot] = {
BOT_KEY_TEMPLATE.format(
name=bot["name"],
homepage=bot["homepage"],
): StoreBot(**bot)
for bot in load_json5_from_web(STORE_BOTS_URL)
for bot in load_json_from_web(STORE_BOTS_URL)
}
self._store_drivers: dict[str, StoreDriver] = {
PYPI_KEY_TEMPLATE.format(
project_link=driver["project_link"],
module_name=driver["module_name"],
): StoreDriver(**driver)
for driver in load_json5_from_web(STORE_DRIVERS_URL)
for driver in load_json_from_web(STORE_DRIVERS_URL)
}
self._store_plugins: dict[str, StorePlugin] = {
PYPI_KEY_TEMPLATE.format(
project_link=plugin["project_link"],
module_name=plugin["module_name"],
): StorePlugin(**plugin)
for plugin in load_json5_from_web(STORE_PLUGINS_URL)
for plugin in load_json_from_web(STORE_PLUGINS_URL)
}
# 上次测试的结果
self._previous_results: dict[str, StoreTestResult] = {
Expand Down
27 changes: 9 additions & 18 deletions src/providers/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,22 @@


def load_json_from_file(file_path: Path):
"""从文件加载 JSON 文件"""
"""从文件加载 JSON5 文件"""
with open(file_path, encoding="utf-8") as file:
return json.load(file)
return pyjson5.decode_io(file) # type: ignore


def load_json_from_web(url: str):
"""从网络加载 JSON 文件"""
"""从网络加载 JSON5 文件"""
r = httpx.get(url)
if r.status_code != 200:
raise ValueError(f"下载文件失败:{r.text}")
return r.json()
return pyjson5.decode(r.text)


def load_json(text: str):
"""从文本加载 JSON5"""
return pyjson5.decode(text)


def dump_json(path: Path, data: Any, minify: bool = True) -> None:
Expand All @@ -34,20 +39,6 @@ def dump_json(path: Path, data: Any, minify: bool = True) -> None:
json.dump(data, f, ensure_ascii=False, indent=2)


def load_json5_from_file(file_path: Path):
"""从文件加载 JSON5 文件"""
with open(file_path, encoding="utf-8") as file:
return pyjson5.decode_io(file) # type: ignore


def load_json5_from_web(url: str):
"""从网络加载 JSON5 文件"""
r = httpx.get(url)
if r.status_code != 200:
raise ValueError(f"下载文件失败:{r.text}")
return pyjson5.decode(r.text)


def dump_json5(path: Path, data: Any) -> None:
"""保存 JSON5 文件

Expand Down
12 changes: 7 additions & 5 deletions src/providers/validation/models.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# ruff: noqa: UP040
import abc
import json
from enum import Enum
from typing import Annotated, Any, TypeAlias

Expand All @@ -17,6 +16,9 @@
)
from pydantic_core import ErrorDetails, PydanticCustomError, to_jsonable_python
from pydantic_extra_types.color import Color
from pyjson5 import Json5DecoderException

from src.providers.utils import load_json

from .constants import (
NAME_MAX_LENGTH,
Expand Down Expand Up @@ -188,8 +190,8 @@ def tags_validator(cls, v: str | list[Any]) -> list[dict[str, str]]:
return v

try:
return json.loads(v)
except json.JSONDecodeError:
return load_json(v)
except Json5DecoderException:
raise PydanticCustomError("json_type", "JSON 格式不合法")


Expand Down Expand Up @@ -236,8 +238,8 @@ def supported_adapters_validator(
# 如果是从 issue 中获取的数据,需要先解码
if skip_test and isinstance(v, str):
try:
v = json.loads(v)
except json.JSONDecodeError:
v = load_json(v)
except Json5DecoderException:
raise PydanticCustomError("json_type", "JSON 格式不合法")

# 如果是支持所有适配器,值应该是 None,不需要检查
Expand Down
4 changes: 2 additions & 2 deletions tests/utils/store_test/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@

async def test_load_json_failed(mocked_api: MockRouter):
"""测试加载 json 失败"""
from src.providers.utils import load_json5_from_web
from src.providers.utils import load_json_from_web

mocked_api.get(STORE_ADAPTERS_URL).respond(404)

with pytest.raises(ValueError, match="下载文件失败:"):
load_json5_from_web(STORE_ADAPTERS_URL)
load_json_from_web(STORE_ADAPTERS_URL)


async def test_get_pypi_data_failed(mocked_api: MockRouter):
Expand Down
Loading