Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Split core functionality and support orjson and msgspec #9

Merged
merged 27 commits into from
May 5, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
60893a5
Refactor into core and json module, add orjson support
nhairs Apr 27, 2024
7f7ea2a
Format tests
nhairs Apr 27, 2024
e9aff5b
Allow tests to complete on all platforms
nhairs Apr 27, 2024
eaf56e2
Fix broken GHA spec
nhairs Apr 27, 2024
c36d6da
Don't support defaults kwargs (py310+ only)
nhairs Apr 27, 2024
4b19678
Drop py37 support, begin testing again py313
nhairs Apr 27, 2024
0d8b8a1
Fix py313 in GHA
nhairs Apr 27, 2024
3897a97
Don't instlal python on pypy
nhairs Apr 27, 2024
2eb8820
Run py313 in py313...
nhairs Apr 27, 2024
bc2b96e
Avoid orjson on python 3.13 while its not supported
nhairs Apr 27, 2024
1441721
Migrate tests to pytest, test OrjsonFormatter where possible
nhairs Apr 28, 2024
c017bef
Update docstrings to use mkdocstrings compatible
nhairs Apr 28, 2024
457a74c
Fix formatting, linting, typing
nhairs Apr 28, 2024
0d2bc65
Remove py37 from tox config
nhairs Apr 28, 2024
0ee6b9a
Maintain backwards compatibility
nhairs Apr 29, 2024
a0b595b
Add more tests
nhairs Apr 30, 2024
77dcdae
Add support for deprecated json.RESERVED_ATTRS
nhairs Apr 30, 2024
b61fc98
fix assert in test
nhairs May 1, 2024
89f0820
Update test names
nhairs May 3, 2024
a2df91b
Add support for msgspec
nhairs May 3, 2024
44c610d
Fix msgspec specifiers
nhairs May 3, 2024
32f763f
simplify ORJSON_AVAILABLE check
nhairs May 3, 2024
487388c
Update README and CHANGELOG
nhairs May 3, 2024
5cc6e3d
Fix formatting
nhairs May 3, 2024
ce76b66
Add other encoders to README
nhairs May 5, 2024
44406d3
Update freezegun issue references
nhairs May 5, 2024
5a6b959
Remove optional dependencies for specific encoders
nhairs May 5, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix formatting, linting, typing
  • Loading branch information
nhairs committed Apr 28, 2024
commit 457a74c3f5a6f81b63ba7135fcf7f1c525187c47
1 change: 1 addition & 0 deletions src/pythonjsonlogger/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
else:
try:
import orjson

ORJSON_AVAILABLE = True
except ImportError:
ORJSON_AVAILABLE = False
1 change: 1 addition & 0 deletions src/pythonjsonlogger/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
Module contains the `JsonFormatter` and a custom `JsonEncoder` which supports a greater
variety of types.
"""

### IMPORTS
### ============================================================================
## Future
Expand Down
72 changes: 44 additions & 28 deletions tests/test_formatters.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import logging
import sys
import traceback
from typing import Any, Generator

## Installed
from freezegun import freeze_time
Expand All @@ -27,12 +28,13 @@

### SETUP
### ============================================================================
ALL_FORMATTERS: list[BaseJsonFormatter] = [JsonFormatter]
ALL_FORMATTERS: list[type[BaseJsonFormatter]] = [JsonFormatter]
if pythonjsonlogger.ORJSON_AVAILABLE:
ALL_FORMATTERS.append(OrjsonFormatter)

_LOGGER_COUNT = 0


@dataclass
class LoggingEnvironment:
logger: logging.Logger
Expand All @@ -48,7 +50,7 @@ def load_json(self) -> Any:


@pytest.fixture
def env() -> LoggingEnvironment:
def env() -> Generator[LoggingEnvironment, None, None]:
global _LOGGER_COUNT # pylint: disable=global-statement
_LOGGER_COUNT += 1
logger = logging.getLogger(f"pythonjsonlogger.tests.{_LOGGER_COUNT}")
Expand All @@ -74,12 +76,13 @@ def get_traceback_from_exception_followed_by_log_call(env_: LoggingEnvironment)
str_traceback = str_traceback[:-1]
return str_traceback


### TESTS
### ============================================================================
## Common Tests
## -----------------------------------------------------------------------------
@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_default_format(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_default_format(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
env.set_formatter(class_())

msg = "testing logging format"
Expand All @@ -92,11 +95,13 @@ def test_default_format(env: LoggingEnvironment, class_: BaseJsonFormatter):


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_percentage_format(env: LoggingEnvironment, class_: BaseJsonFormatter):
env.set_formatter(class_(
# All kind of different styles to check the regex
"[%(levelname)8s] %(message)s %(filename)s:%(lineno)d %(asctime)"
))
def test_percentage_format(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
env.set_formatter(
class_(
# All kind of different styles to check the regex
"[%(levelname)8s] %(message)s %(filename)s:%(lineno)d %(asctime)"
)
)

msg = "testing logging format"
env.logger.info(msg)
Expand All @@ -108,7 +113,7 @@ def test_percentage_format(env: LoggingEnvironment, class_: BaseJsonFormatter):


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_rename_base_field(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_rename_base_field(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
env.set_formatter(class_(rename_fields={"message": "@message"}))

msg = "testing logging format"
Expand All @@ -120,7 +125,7 @@ def test_rename_base_field(env: LoggingEnvironment, class_: BaseJsonFormatter):


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_rename_nonexistent_field(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_rename_nonexistent_field(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
env.set_formatter(class_(rename_fields={"nonexistent_key": "new_name"}))

stderr_watcher = io.StringIO()
Expand All @@ -133,7 +138,7 @@ def test_rename_nonexistent_field(env: LoggingEnvironment, class_: BaseJsonForma


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_add_static_fields(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_add_static_fields(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
env.set_formatter(class_(static_fields={"log_stream": "kafka"}))

msg = "testing static fields"
Expand All @@ -146,7 +151,7 @@ def test_add_static_fields(env: LoggingEnvironment, class_: BaseJsonFormatter):


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_format_keys(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_format_keys(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
supported_keys = [
"asctime",
"created",
Expand Down Expand Up @@ -182,15 +187,15 @@ def test_format_keys(env: LoggingEnvironment, class_: BaseJsonFormatter):


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_unknown_format_key(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_unknown_format_key(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
env.set_formatter(class_("%(unknown_key)s %(message)s"))
env.logger.info("testing unknown logging format")
# make sure no error occurs
return


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_log_dict(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_log_dict(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
env.set_formatter(class_())

msg = {"text": "testing logging", "num": 1, 5: "9", "nested": {"more": "data"}}
Expand All @@ -199,18 +204,18 @@ def test_log_dict(env: LoggingEnvironment, class_: BaseJsonFormatter):

assert log_json["text"] == msg["text"]
assert log_json["num"] == msg["num"]
assert log_json["5"] == msg[5]
assert log_json["5"] == msg[5]
assert log_json["nested"] == msg["nested"]
assert log_json["message"] == ""
return


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_log_extra(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_log_extra(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
env.set_formatter(class_())

extra = {"text": "testing logging", "num": 1, 5: "9", "nested": {"more": "data"}}
env.logger.info("hello", extra=extra)
env.logger.info("hello", extra=extra) # type: ignore[arg-type]
log_json = env.load_json()

assert log_json["text"] == extra["text"]
Expand All @@ -220,9 +225,10 @@ def test_log_extra(env: LoggingEnvironment, class_: BaseJsonFormatter):
assert log_json["message"] == "hello"
return


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_json_custom_logic_adds_field(env: LoggingEnvironment, class_: BaseJsonFormatter):
class CustomJsonFormatter(class_):
def test_json_custom_logic_adds_field(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
class CustomJsonFormatter(class_): # type: ignore[valid-type,misc]

def process_log_record(self, log_record):
log_record["custom"] = "value"
Expand All @@ -237,7 +243,7 @@ def process_log_record(self, log_record):


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_exc_info(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_exc_info(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
env.set_formatter(class_())

expected_value = get_traceback_from_exception_followed_by_log_call(env)
Expand All @@ -246,8 +252,9 @@ def test_exc_info(env: LoggingEnvironment, class_: BaseJsonFormatter):
assert log_json["exc_info"] == expected_value
return


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_exc_info_renamed(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_exc_info_renamed(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
env.set_formatter(class_("%(exc_info)s", rename_fields={"exc_info": "stack_trace"}))

expected_value = get_traceback_from_exception_followed_by_log_call(env)
Expand All @@ -257,23 +264,25 @@ def test_exc_info_renamed(env: LoggingEnvironment, class_: BaseJsonFormatter):
assert "exc_info" not in log_json
return


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_custom_object_serialization(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_custom_object_serialization(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
def encode_complex(z):
if isinstance(z, complex):
return (z.real, z.imag)
raise TypeError(f"Object of type {type(z)} is no JSON serializable")

env.set_formatter(class_(json_default=encode_complex))
env.set_formatter(class_(json_default=encode_complex)) # type: ignore[call-arg]

env.logger.info("foo", extra={"special": complex(3, 8)})
log_json = env.load_json()

assert log_json["special"] == [3.0, 8.0]
return


@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_rename_reserved_attrs(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_rename_reserved_attrs(env: LoggingEnvironment, class_: type[BaseJsonFormatter]):
log_format = lambda x: [f"%({i:s})s" for i in x]
reserved_attrs_map = {
"exc_info": "error.type",
Expand All @@ -290,9 +299,9 @@ def test_rename_reserved_attrs(env: LoggingEnvironment, class_: BaseJsonFormatte
reserved_attrs = [
attr for attr in RESERVED_ATTRS if attr not in list(reserved_attrs_map.keys())
]
env.set_formatter(class_(
custom_format, reserved_attrs=reserved_attrs, rename_fields=reserved_attrs_map
))
env.set_formatter(
class_(custom_format, reserved_attrs=reserved_attrs, rename_fields=reserved_attrs_map)
)

env.logger.info("message")
log_json = env.load_json()
Expand All @@ -306,9 +315,12 @@ def test_rename_reserved_attrs(env: LoggingEnvironment, class_: BaseJsonFormatte
assert old_name not in log_json
return


@freeze_time(datetime.datetime(2017, 7, 14, 2, 40))
@pytest.mark.parametrize("class_", ALL_FORMATTERS)
def test_json_default_encoder_with_timestamp(env: LoggingEnvironment, class_: BaseJsonFormatter):
def test_json_default_encoder_with_timestamp(
env: LoggingEnvironment, class_: type[BaseJsonFormatter]
):
if pythonjsonlogger.ORJSON_AVAILABLE and class_ is OrjsonFormatter:
# https://github.com/spulec/freezegun/issues/448#issuecomment-1686070438
pytest.xfail()
Expand All @@ -322,6 +334,7 @@ def test_json_default_encoder_with_timestamp(env: LoggingEnvironment, class_: Ba
assert log_json["timestamp"] == "2017-07-14T02:40:00+00:00"
return


## JsonFormatter Specific
## -----------------------------------------------------------------------------
def test_json_default_encoder(env: LoggingEnvironment):
Expand All @@ -342,6 +355,7 @@ def test_json_default_encoder(env: LoggingEnvironment):
assert log_json["otherdatetimeagain"] == "1900-01-01T00:00:00"
return


def test_json_custom_default(env: LoggingEnvironment):
def custom(o):
return "very custom"
Expand All @@ -356,6 +370,7 @@ def custom(o):
assert log_json["normal"] == "value"
return


def test_ensure_ascii_true(env: LoggingEnvironment):
env.set_formatter(JsonFormatter())
env.logger.info("Привет")
Expand All @@ -365,6 +380,7 @@ def test_ensure_ascii_true(env: LoggingEnvironment):
assert msg, r"\u041f\u0440\u0438\u0432\u0435\u0442"
return


def test_ensure_ascii_false(env: LoggingEnvironment):
env.set_formatter(JsonFormatter(json_ensure_ascii=False))
env.logger.info("Привет")
Expand Down
Loading