Skip to content

Commit

Permalink
Set default of JSON encoder to ensure_ascii = False (#154)
Browse files Browse the repository at this point in the history
* Update default for ensure_ascii in JSON encoder
* Update doc-strings
  • Loading branch information
eyurtsev authored May 7, 2023
1 parent 5e40290 commit 0088b03
Showing 1 changed file with 14 additions and 20 deletions.
34 changes: 14 additions & 20 deletions kor/encoders/json_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,34 +25,28 @@ class JSONEncoder(Encoder):
from kor import JSONEncoder
json_encoder = JSONEncoder(use_tags=True)
json_encoder.encode({"object": [{"a": 1}]})
# '<json>{"object": [{"a": 1}]}</json>'
json_encoder = JSONEncoder(use_tags=True, ensure_ascii=False)
data = {"name": "Café"}
json_encoder.encode(data)
# '<json>{"name": "Café"}</json>'
json_encoder = JSONEncoder(use_tags=True, ensure_ascii=True)
data = {"name": "Café"}
json_encoder.encode(data)
# '<json>{"name": "Caf\\u00e9"}</json>'
"""

def __init__(self, use_tags: bool = True, ensure_ascii: bool = True) -> None:
def __init__(self, use_tags: bool = True, ensure_ascii: bool = False) -> None:
"""Initialize the JSON encoder.
Args:
use_tags: Whether to wrap the output in a special JSON tags.
This may help identify the JSON content in cases when
the model attempts to add clarifying explanations.
ensure_ascii: Whether to escape non-ASCII characters.
data = {"name": "Café"}
# Using ensure_ascii=True (default)
json_str = json.dumps(data)
print(json_str) # {"name": "Caf\u00e9"}
# Using ensure_ascii=False
json_str = json.dumps(data, ensure_ascii=False)
print(json_str) # {"name": "Café"}
Args:
use_tags: Whether to wrap the output in a special JSON tags.
This may help identify the JSON content in cases when
the model attempts to add clarifying explanations.
ensure_ascii: Whether to escape non-ASCII characters.
Default is False to preserve non-ASCII characters as
that it a more sensible behavior for the extraction
use cases.
"""
self.use_tags = use_tags
self.ensure_ascii = ensure_ascii
Expand Down

0 comments on commit 0088b03

Please sign in to comment.